internal/releasetargets: build all ports

Now that we're cross-compiling everything, we can build all ports
(except ios/android/js, of course). Add a script that uses the gotip
command to populate a list of ports for the latest development version.

Change-Id: I7afdeef739934a8dee79cec033f1782b661cb848
Reviewed-on: https://go-review.googlesource.com/c/build/+/479278
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Auto-Submit: Heschi Kreinick <heschi@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Carlos Amedee <carlos@golang.org>
Run-TryBot: Heschi Kreinick <heschi@google.com>

cmd/securitybot/main.go

@@ -151,7 +151,7 @@ func (t *tester) runTests(ctx context.Context, builderType string, info *buildIn
 		return builderResult{builderType: builderType, err: fmt.Errorf("failed to get work dir: %s", err)}
 	}
 
-	env := append(buildConfig.Env(), "GOPATH="+work+"/gopath", "GOROOT_FINAL="+buildConfig.GorootFinal(), "GOROOT="+work+"/go")
+	env := append(buildConfig.Env(), "GOPATH="+work+"/gopath", "GOROOT_FINAL="+dashboard.GorootFinal(buildConfig.GOOS()), "GOROOT="+work+"/go")
 	// Because we are unable to determine the internal GCE hostname of the
 	// coordinator, we cannot use the same GOPROXY proxy that the public TryBots
 	// use to get around the disabled network. Instead of using that proxy

dashboard/builders.go

@@ -1327,9 +1327,9 @@ func (c *BuildConfig) MakeScriptArgs() []string {
 }
 
 // GorootFinal returns the default install location for
-// releases for this platform.
-func (c *BuildConfig) GorootFinal() string {
-	if strings.HasPrefix(c.Name, "windows-") {
+// releases for the given GOOS.
+func GorootFinal(goos string) string {
+	if goos == "windows" {
 		return "c:\\go"
 	}
 	return "/usr/local/go"

internal/releasetargets/allports/go1.21.txt

@@ -0,0 +1,46 @@
+aix/ppc64
+android/386
+android/amd64
+android/arm
+android/arm64
+darwin/amd64
+darwin/arm64
+dragonfly/amd64
+freebsd/386
+freebsd/amd64
+freebsd/arm
+freebsd/arm64
+freebsd/riscv64
+illumos/amd64
+ios/amd64
+ios/arm64
+js/wasm
+linux/386
+linux/amd64
+linux/arm
+linux/arm64
+linux/loong64
+linux/mips
+linux/mips64
+linux/mips64le
+linux/mipsle
+linux/ppc64
+linux/ppc64le
+linux/riscv64
+linux/s390x
+netbsd/386
+netbsd/amd64
+netbsd/arm
+netbsd/arm64
+openbsd/386
+openbsd/amd64
+openbsd/arm
+openbsd/arm64
+plan9/386
+plan9/amd64
+plan9/arm
+solaris/amd64
+windows/386
+windows/amd64
+windows/arm
+windows/arm64

internal/releasetargets/genlatestports.bash

@@ -0,0 +1,7 @@
+#!/bin/bash -eux
+
+go install golang.org/dl/gotip@latest
+gotip download
+MAJOR=$(gotip version | grep -Eo 'go1\.[0-9]+')
+gotip tool dist list > allports/${MAJOR}.txt
+

internal/releasetargets/releases.txt

@@ -153,18 +153,30 @@ windows-arm64   windows    arm64      windows-arm64-11
 
 Targets for release 1.21
 ================================================================================
+aix-ppc64       aix        ppc64      (cross-compiled, no tests)
+
 darwin-amd64    darwin     amd64      darwin-amd64-13
 	Running on reverse builder: AWS macOS Ventura (13) VM under QEMU
 
 darwin-arm64    darwin     arm64      darwin-arm64-12
 	Running on reverse builder: macOS Monterey (12) ARM64 (M1) on Mac minis in a Google office
 
+dragonfly-amd64 dragonfly  amd64      (cross-compiled, no tests)
+
 freebsd-386     freebsd    386        freebsd-386-13_0
 	Running on GCP
 
 freebsd-amd64   freebsd    amd64      freebsd-amd64-13_0
 	Running on GCP
 
+freebsd-arm     freebsd    arm        (cross-compiled, no tests)
+
+freebsd-arm64   freebsd    arm64      (cross-compiled, no tests)
+
+freebsd-riscv64 freebsd    riscv64    (cross-compiled, no tests)
+
+illumos-amd64   illumos    amd64      (cross-compiled, no tests)
+
 linux-386       linux      386        linux-386-bullseye
 	Long tests on linux-386-longtest
 	Running on GCP
@@ -173,19 +185,55 @@ linux-amd64     linux      amd64      linux-amd64-bullseye
 	Long tests on linux-amd64-longtest
 	Running on GCP
 
+linux-arm       linux      arm        (cross-compiled, no tests)
+
 linux-arm64     linux      arm64      linux-arm64
 	Running on GCP
 
 linux-armv6l    linux      arm        linux-arm-aws
 	Running on AWS
 
-linux-ppc64le   linux      ppc64le    linux-ppc64le-buildlet
-	Build only
-	Running on reverse builder: Ubuntu 20.04; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl
+linux-loong64   linux      loong64    (cross-compiled, no tests)
 
-linux-s390x     linux      s390x      linux-s390x-crosscompile
-	Build only
-	Running on GCP
+linux-mips      linux      mips       (cross-compiled, no tests)
+
+linux-mips64    linux      mips64     (cross-compiled, no tests)
+
+linux-mips64le  linux      mips64le   (cross-compiled, no tests)
+
+linux-mipsle    linux      mipsle     (cross-compiled, no tests)
+
+linux-ppc64     linux      ppc64      (cross-compiled, no tests)
+
+linux-ppc64le   linux      ppc64le    (cross-compiled, no tests)
+
+linux-riscv64   linux      riscv64    (cross-compiled, no tests)
+
+linux-s390x     linux      s390x      (cross-compiled, no tests)
+
+netbsd-386      netbsd     386        (cross-compiled, no tests)
+
+netbsd-amd64    netbsd     amd64      (cross-compiled, no tests)
+
+netbsd-arm      netbsd     arm        (cross-compiled, no tests)
+
+netbsd-arm64    netbsd     arm64      (cross-compiled, no tests)
+
+openbsd-386     openbsd    386        (cross-compiled, no tests)
+
+openbsd-amd64   openbsd    amd64      (cross-compiled, no tests)
+
+openbsd-arm     openbsd    arm        (cross-compiled, no tests)
+
+openbsd-arm64   openbsd    arm64      (cross-compiled, no tests)
+
+plan9-386       plan9      386        (cross-compiled, no tests)
+
+plan9-amd64     plan9      amd64      (cross-compiled, no tests)
+
+plan9-arm       plan9      arm        (cross-compiled, no tests)
+
+solaris-amd64   solaris    amd64      (cross-compiled, no tests)
 
 windows-386     windows    386        windows-386-2008
 	Running on GCP
@@ -194,6 +242,10 @@ windows-amd64   windows    amd64      windows-amd64-2008
 	Long tests on windows-amd64-longtest
 	Running on GCP
 
+windows-arm     windows    arm        windows-arm64-11
+	Build only
+	Running on reverse builder: Azure windows 11 arm64 VMs
+
 windows-arm64   windows    arm64      windows-arm64-11
 	Running on reverse builder: Azure windows 11 arm64 VMs
 

internal/releasetargets/releasetargets.go

@@ -5,7 +5,9 @@
 package releasetargets
 
 import (
+	"embed"
 	"fmt"
+	"io/fs"
 	"sort"
 	"strings"
 	"sync"
@@ -35,6 +37,10 @@ type OSArch struct {
 	OS, Arch string
 }
 
+func (o OSArch) String() string {
+	return o.OS + "-" + o.Arch
+}
+
 func (rt ReleaseTargets) FirstClassPorts() map[OSArch]bool {
 	result := map[OSArch]bool{}
 	for _, target := range rt {
@@ -47,8 +53,8 @@ func (rt ReleaseTargets) FirstClassPorts() map[OSArch]bool {
 
 // allReleases contains all the targets for all releases we're currently
 // supporting. To reduce duplication, targets from earlier versions are
-// propagated forward unless overridden. To remove a target in a later release,
-// set it to nil explicitly.
+// propagated forward unless overridden. To stop configuring a target in a
+// later release, set it to nil explicitly.
 // GOOS and GOARCH will be set automatically from the target name, but can be
 // overridden if necessary. Name will also be set and should not be overridden.
 var allReleases = map[int]ReleaseTargets{
@@ -164,12 +170,27 @@ var allReleases = map[int]ReleaseTargets{
 			Builder:         "darwin-amd64-13",
 			MinMacOSVersion: "10.15", // go.dev/issue/57125
 		},
+		"linux-s390x":   nil,
+		"linux-ppc64le": nil,
+		"windows-arm": &Target{
+			Builder:     "windows-arm64-11", // Windows builds need a builder to create their MSIs.
+			SecondClass: true,
+			BuildOnly:   true,
+		},
 	},
 }
 
+//go:generate ./genlatestports.bash
+//go:embed allports/*.txt
+var allPortsFS embed.FS
+var allPorts = map[int][]OSArch{}
+
 func init() {
 	for _, targets := range allReleases {
 		for name, target := range targets {
+			if target == nil {
+				continue
+			}
 			if target.Name != "" {
 				panic(fmt.Sprintf("target.Name in %q should be left inferred", name))
 			}
@@ -187,6 +208,24 @@ func init() {
 			}
 		}
 	}
+	files, err := allPortsFS.ReadDir("allports")
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range files {
+		major := 0
+		if n, err := fmt.Sscanf(f.Name(), "go1.%d.txt", &major); err != nil || n == 0 {
+			panic("failed to parse filename " + f.Name())
+		}
+		body, err := fs.ReadFile(allPortsFS, "allports/"+f.Name())
+		if err != nil {
+			panic(err)
+		}
+		for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") {
+			os, arch, _ := strings.Cut(line, "/")
+			allPorts[major] = append(allPorts[major], OSArch{os, arch})
+		}
+	}
 }
 
 func sortedReleases() []int {
@@ -198,10 +237,17 @@ func sortedReleases() []int {
 	return releases
 }
 
+var unbuildableOSs = map[string]bool{
+	"android": true,
+	"ios":     true,
+	"js":      true,
+}
+
 // TargetsForGo1Point returns the ReleaseTargets that apply to the given
 // version.
 func TargetsForGo1Point(x int) ReleaseTargets {
 	targets := ReleaseTargets{}
+	var ports []OSArch
 	for _, release := range sortedReleases() {
 		if release > x {
 			break
@@ -214,6 +260,23 @@ func TargetsForGo1Point(x int) ReleaseTargets {
 				targets[osarch] = &copy
 			}
 		}
+		if p, ok := allPorts[release]; ok {
+			ports = p
+		}
+	}
+	for _, osarch := range ports {
+		_, unbuildable := unbuildableOSs[osarch.OS]
+		_, exists := targets[osarch.String()]
+		if unbuildable || exists {
+			continue
+		}
+		targets[osarch.String()] = &Target{
+			Name:        osarch.String(),
+			GOOS:        osarch.OS,
+			GOARCH:      osarch.Arch,
+			SecondClass: true,
+			BuildOnly:   true,
+		}
 	}
 	return targets
 }

internal/releasetargets/releasetargets_test.go

@@ -54,8 +54,14 @@ func printRelease(w io.Writer, release int, targets ReleaseTargets) {
 	for _, name := range targetNames {
 		target := targets[name]
 		var flags []string
+		builder := target.Builder
 		if target.BuildOnly {
-			flags = append(flags, "Build only")
+			if builder == "" {
+				builder = "(cross-compiled, no tests)"
+			} else {
+				flags = append(flags, "Build only")
+			}
+
 		}
 		if target.Race {
 			flags = append(flags, "Race enabled")
@@ -63,7 +69,7 @@ func printRelease(w io.Writer, release int, targets ReleaseTargets) {
 		if target.LongTestBuilder != "" {
 			flags = append(flags, "Long tests on "+target.LongTestBuilder)
 		}
-		fmt.Fprintf(w, "%-15v %-10v %-10v %v\n", name, target.GOOS, target.GOARCH, target.Builder)
+		fmt.Fprintf(w, "%-15v %-10v %-10v %v\n", name, target.GOOS, target.GOARCH, builder)
 		if len(flags) != 0 {
 			fmt.Fprintf(w, "\t%v\n", strings.Join(flags, ", "))
 		}
@@ -93,6 +99,9 @@ func printRelease(w io.Writer, release int, targets ReleaseTargets) {
 func TestBuildersExist(t *testing.T) {
 	for _, rel := range allReleases {
 		for _, target := range rel {
+			if target == nil || target.Builder == "" {
+				continue
+			}
 			_, ok := dashboard.Builders[target.Builder]
 			if !ok {
 				t.Errorf("missing builder: %q", target.Builder)

internal/task/buildrelease.go

@@ -345,10 +345,8 @@ func (b *BuildletStep) BuildBinary(ctx *workflow.TaskContext, sourceArchive io.R
 }
 
 func (b *BuildletStep) makeEnv() []string {
-	// Use the target build config for distpack builds.
-	bc := dashboard.Builders[b.Target.Builder]
 	// We need GOROOT_FINAL both during the binary build and test runs. See go.dev/issue/52236.
-	makeEnv := []string{"GOROOT_FINAL=" + bc.GorootFinal()}
+	makeEnv := []string{"GOROOT_FINAL=" + dashboard.GorootFinal(b.Target.GOOS)}
 	// Add extra vars from the target's configuration.
 	makeEnv = append(makeEnv, b.Target.ExtraEnv...)
 	return makeEnv