From 3a7ac76259cf26802a5bf2cf8842a307bdeecebc Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Wed, 30 Nov 2022 16:50:25 -0600 Subject: [PATCH] all: add linux-ppc64le-power10-osu builder This builder requires a patched bootstrap toolchain to accept GOPPC64=power10. Likewise, this folds in a change to update the container images to ubuntu 20.04. This is and has been running on the existing ppc64le builders for a couple months already. Change-Id: Ia86cb711c99c0d8ab5f9de0ae50129fe11e6ccab Reviewed-on: https://go-review.googlesource.com/c/build/+/454515 Run-TryBot: Paul Murphy TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt Reviewed-by: Carlos Amedee Auto-Submit: Carlos Amedee --- cmd/buildlet/stage0/stage0.go | 2 + dashboard/builders.go | 164 ++++++++++++++++------------ env/linux-ppc64le/osuosl/Dockerfile | 27 ++++- env/linux-ppc64le/osuosl/Makefile | 3 + env/linux-ppc64le/osuosl/NOTES | 5 + env/linux-ppc64le/osuosl/p10.patch | 16 +++ 6 files changed, 139 insertions(+), 78 deletions(-) create mode 100644 env/linux-ppc64le/osuosl/p10.patch diff --git a/cmd/buildlet/stage0/stage0.go b/cmd/buildlet/stage0/stage0.go index 1b158e17..1f3e2dbe 100644 --- a/cmd/buildlet/stage0/stage0.go +++ b/cmd/buildlet/stage0/stage0.go @@ -171,6 +171,8 @@ Download: cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...) cmd.Args = append(cmd.Args, os.ExpandEnv("--workdir=${WORKDIR}")) cmd.Args = append(cmd.Args, os.ExpandEnv("--hostname=${GO_BUILDER_ENV}")) + case "host-linux-ppc64le-power10-osu": + cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...) case "host-linux-ppc64le-power9-osu": cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...) case "host-linux-ppc64le-osu": // power8 diff --git a/dashboard/builders.go b/dashboard/builders.go index 6782e0a9..fc7e60ac 100644 --- a/dashboard/builders.go +++ b/dashboard/builders.go @@ -28,77 +28,79 @@ var slowBotAliases = map[string]string{ // Known missing builders: "ios-amd64": "", // There is no builder for the iOS Simulator. See issues 42100 and 42177. - "386": "linux-386", - "aix": "aix-ppc64", - "amd64": "linux-amd64", - "android": "android-amd64-emu", - "android-386": "android-386-emu", - "android-amd64": "android-amd64-emu", - "android-arm": "android-arm-corellium", - "android-arm64": "android-arm64-corellium", - "arm": "linux-arm-aws", - "arm64": "linux-arm64", - "darwin": "darwin-amd64-12_0", - "darwin-amd64": "darwin-amd64-12_0", - "darwin-arm64": "darwin-arm64-12", - "ios-arm64": "ios-arm64-corellium", - "dragonfly": "dragonfly-amd64-622", - "dragonfly-amd64": "dragonfly-amd64-622", - "freebsd": "freebsd-amd64-13_0", - "freebsd-386": "freebsd-386-13_0", - "freebsd-amd64": "freebsd-amd64-13_0", - "freebsd-arm": "freebsd-arm-paulzhol", - "freebsd-arm64": "freebsd-arm64-dmgk", - "freebsd-riscv64": "freebsd-riscv64-unmatched", - "illumos": "illumos-amd64", - "ios": "ios-arm64-corellium", - "js": "js-wasm", - "linux": "linux-amd64", - "linux-arm": "linux-arm-aws", - "linux-loong64": "linux-loong64-3a5000", - "linux-mips": "linux-mips-rtrk", - "linux-mips64": "linux-mips64-rtrk", - "linux-mips64le": "linux-mips64le-rtrk", - "linux-mipsle": "linux-mipsle-rtrk", - "linux-ppc64": "linux-ppc64-sid-buildlet", - "linux-ppc64le": "linux-ppc64le-buildlet", - "linux-ppc64le-power9": "linux-ppc64le-power9osu", - "linux-riscv64": "linux-riscv64-unmatched", - "linux-s390x": "linux-s390x-ibm", - "longtest": "linux-amd64-longtest", - "loong64": "linux-loong64-3a5000", - "mips": "linux-mips-rtrk", - "mips64": "linux-mips64-rtrk", - "mips64le": "linux-mips64le-rtrk", - "mipsle": "linux-mipsle-rtrk", - "netbsd": "netbsd-amd64-9_3", - "netbsd-386": "netbsd-386-9_3", - "netbsd-amd64": "netbsd-amd64-9_3", - "netbsd-arm": "netbsd-arm-bsiegert", - "netbsd-arm64": "netbsd-arm64-bsiegert", - "nocgo": "linux-amd64-nocgo", - "openbsd": "openbsd-amd64-70", - "openbsd-386": "openbsd-386-70", - "openbsd-amd64": "openbsd-amd64-70", - "openbsd-arm": "openbsd-arm-jsing", - "openbsd-arm64": "openbsd-arm64-jsing", - "openbsd-mips64": "openbsd-mips64-jsing", - "plan9": "plan9-arm", - "plan9-386": "plan9-386-0intro", - "plan9-amd64": "plan9-amd64-0intro", - "ppc64": "linux-ppc64-sid-buildlet", - "ppc64le": "linux-ppc64le-buildlet", - "ppc64lep9": "linux-ppc64le-power9osu", - "riscv64": "linux-riscv64-unmatched", - "s390x": "linux-s390x-ibm", - "solaris": "solaris-amd64-oraclerel", - "solaris-amd64": "solaris-amd64-oraclerel", - "wasm": "js-wasm", - "windows": "windows-amd64-2016", - "windows-386": "windows-386-2008", - "windows-amd64": "windows-amd64-2016", - "windows-arm": "windows-arm-zx2c4", - "windows-arm64": "windows-arm64-11", + "386": "linux-386", + "aix": "aix-ppc64", + "amd64": "linux-amd64", + "android": "android-amd64-emu", + "android-386": "android-386-emu", + "android-amd64": "android-amd64-emu", + "android-arm": "android-arm-corellium", + "android-arm64": "android-arm64-corellium", + "arm": "linux-arm-aws", + "arm64": "linux-arm64", + "darwin": "darwin-amd64-12_0", + "darwin-amd64": "darwin-amd64-12_0", + "darwin-arm64": "darwin-arm64-12", + "ios-arm64": "ios-arm64-corellium", + "dragonfly": "dragonfly-amd64-622", + "dragonfly-amd64": "dragonfly-amd64-622", + "freebsd": "freebsd-amd64-13_0", + "freebsd-386": "freebsd-386-13_0", + "freebsd-amd64": "freebsd-amd64-13_0", + "freebsd-arm": "freebsd-arm-paulzhol", + "freebsd-arm64": "freebsd-arm64-dmgk", + "freebsd-riscv64": "freebsd-riscv64-unmatched", + "illumos": "illumos-amd64", + "ios": "ios-arm64-corellium", + "js": "js-wasm", + "linux": "linux-amd64", + "linux-arm": "linux-arm-aws", + "linux-loong64": "linux-loong64-3a5000", + "linux-mips": "linux-mips-rtrk", + "linux-mips64": "linux-mips64-rtrk", + "linux-mips64le": "linux-mips64le-rtrk", + "linux-mipsle": "linux-mipsle-rtrk", + "linux-ppc64": "linux-ppc64-sid-buildlet", + "linux-ppc64le": "linux-ppc64le-buildlet", + "linux-ppc64le-power9": "linux-ppc64le-power9osu", + "linux-ppc64le-power10": "linux-ppc64le-power10osu", + "linux-riscv64": "linux-riscv64-unmatched", + "linux-s390x": "linux-s390x-ibm", + "longtest": "linux-amd64-longtest", + "loong64": "linux-loong64-3a5000", + "mips": "linux-mips-rtrk", + "mips64": "linux-mips64-rtrk", + "mips64le": "linux-mips64le-rtrk", + "mipsle": "linux-mipsle-rtrk", + "netbsd": "netbsd-amd64-9_3", + "netbsd-386": "netbsd-386-9_3", + "netbsd-amd64": "netbsd-amd64-9_3", + "netbsd-arm": "netbsd-arm-bsiegert", + "netbsd-arm64": "netbsd-arm64-bsiegert", + "nocgo": "linux-amd64-nocgo", + "openbsd": "openbsd-amd64-70", + "openbsd-386": "openbsd-386-70", + "openbsd-amd64": "openbsd-amd64-70", + "openbsd-arm": "openbsd-arm-jsing", + "openbsd-arm64": "openbsd-arm64-jsing", + "openbsd-mips64": "openbsd-mips64-jsing", + "plan9": "plan9-arm", + "plan9-386": "plan9-386-0intro", + "plan9-amd64": "plan9-amd64-0intro", + "ppc64": "linux-ppc64-sid-buildlet", + "ppc64le": "linux-ppc64le-buildlet", + "ppc64lep9": "linux-ppc64le-power9osu", + "ppc64lep10": "linux-ppc64le-power10osu", + "riscv64": "linux-riscv64-unmatched", + "s390x": "linux-s390x-ibm", + "solaris": "solaris-amd64-oraclerel", + "solaris-amd64": "solaris-amd64-oraclerel", + "wasm": "js-wasm", + "windows": "windows-amd64-2016", + "windows-386": "windows-386-2008", + "windows-amd64": "windows-amd64-2016", + "windows-arm": "windows-arm-zx2c4", + "windows-arm64": "windows-arm64-11", } // Builders are the different build configurations. @@ -389,6 +391,17 @@ var Hosts = map[string]*HostConfig{ SSHUsername: "root", HermeticReverse: true, }, + "host-linux-ppc64le-power10-osu": { + Notes: "Ubuntu 20.04; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl", + Owners: []*gophers.Person{gh("pmur")}, + IsReverse: true, + // GOPPC64=power10 is only supported in go1.20 and later. The container provides a patched boostrap compiler. + env: []string{ + "GOPPC64=power10", + "GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"}, + SSHUsername: "root", + HermeticReverse: true, + }, "host-linux-ppc64le-power9-osu": { Notes: "Ubuntu 20.04; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl", Owners: []*gophers.Person{gh("pmur")}, @@ -2638,6 +2651,13 @@ func init() { distTestAdjust: ppc64DistTestPolicy, env: []string{"GO_TEST_TIMEOUT_SCALE=2"}, // see go.dev/issues/44422 }) + addBuilder(BuildConfig{ + Name: "linux-ppc64le-power10osu", + HostType: "host-linux-ppc64le-power10-osu", + FlakyNet: true, + distTestAdjust: ppc64DistTestPolicy, + env: []string{"GO_TEST_TIMEOUT_SCALE=2"}, // see go.dev/issues/44422 + }) addBuilder(BuildConfig{ Name: "linux-arm64-aws", HostType: "host-linux-arm64-aws", @@ -3044,7 +3064,7 @@ func noTestDirAndNoReboot(run bool, distTest string, isNormalTry bool) bool { } // ppc64DistTestPolicy is a distTestAdjust policy function -// that's shared by linux-ppc64le, -ppc64le-power9osu, and -ppc64. +// that's shared by linux-ppc64le, -ppc64le-power{9,10}-osu, and -ppc64. func ppc64DistTestPolicy(run bool, distTest string, isNormalTry bool) bool { if distTest == "reboot" { // Skip test. It seems to use a lot of memory? diff --git a/env/linux-ppc64le/osuosl/Dockerfile b/env/linux-ppc64le/osuosl/Dockerfile index 40bd5c48..041884e7 100644 --- a/env/linux-ppc64le/osuosl/Dockerfile +++ b/env/linux-ppc64le/osuosl/Dockerfile @@ -2,20 +2,35 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -FROM debian:buster +FROM ubuntu:20.04 ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ apt-get install --yes \ - gcc curl strace \ + gcc gdb curl strace \ ca-certificates netbase \ procps lsof psmisc \ - openssh-server + openssh-server \ + patch -RUN mkdir /usr/local/go-bootstrap && \ - curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64le.tar.gz | \ - tar -C /usr/local/go-bootstrap -zxv +# For GOPPC64=power10 support. This hack can be removed once the bootstrap +# minimum reachs go1.20. +# P8 and P9 will use the default bootstrap toolchain downloaded during CI. +ADD p10.patch / +RUN mkdir -p /usr/local && \ + curl --silent -L https://go.dev/dl/go1.17.13.linux-ppc64le.tar.gz --output /go.tar.gz && \ + tar -C /usr/local/ -zxf /go.tar.gz && \ + mv /usr/local/go /usr/local/go-bootstrap && \ + tar -C /usr/local/ -zxf /go.tar.gz && \ + mv /usr/local/go /usr/local/go-tmp && \ + cd /usr/local/go-bootstrap && patch -p1 < /p10.patch && \ + cd /usr/local/go-bootstrap/src && GOROOT_BOOTSTRAP=/usr/local/go-tmp GOOS=linux GOARCH=ppc64le ./bootstrap.bash && \ + /usr/local/go-tmp/bin/go install golang.org/x/build/cmd/genbootstrap@v0.0.0-20221130230848-98eb5685978b && \ + cd /usr/local && /root/go/bin/genbootstrap -v -skip_build go-linux-ppc64le-bootstrap linux-ppc64le && \ + rm -rf /usr/local/go-tmp /usr/local/go-bootstrap /usr/local/go-linux-ppc64le-bootstrap /usr/local/go-linux-ppc64le-bootstrap.tbz && \ + mkdir /usr/local/go-bootstrap && tar -C /usr/local/go-bootstrap -xf /tmp/genbootstrap-*/gobootstrap-linux-ppc64le-go*.tar.gz && \ + rm -rf /usr/local/go-tmp /root/go /root/.cache /p10.patch /go.tar.gz /tmp/* && apt-get remove --yes patch ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap ADD stage0 /usr/local/bin/stage0 diff --git a/env/linux-ppc64le/osuosl/Makefile b/env/linux-ppc64le/osuosl/Makefile index d3d1245f..67cf728b 100644 --- a/env/linux-ppc64le/osuosl/Makefile +++ b/env/linux-ppc64le/osuosl/Makefile @@ -3,3 +3,6 @@ power8: power9: ./setup-remote.sh ubuntu@140.211.169.171 host-linux-ppc64le-power9-osu + +power10: + ./setup-remote.sh ubuntu@go-le-p10-1.osuosl.org host-linux-ppc64le-power10-osu diff --git a/env/linux-ppc64le/osuosl/NOTES b/env/linux-ppc64le/osuosl/NOTES index 32b42ab6..e9920ded 100644 --- a/env/linux-ppc64le/osuosl/NOTES +++ b/env/linux-ppc64le/osuosl/NOTES @@ -8,6 +8,11 @@ Filing tickets: https://support.osuosl.org/ Ticket password: go/pw-osuosl-ppc64 Machines: +* go-le-p10-1: (30 GB RAM, 50 GB disk, 20 cores, POWER10) + Linux go-le-p10-1 5.15.0-53-generic #59-Ubuntu SMP Mon Oct 17 18:55:27 UTC 2022 ppc64le ppc64le ppc64le GNU/Linux + + ssh -i ~/.ssh/id_ed25519_golang1 ubuntu@go-le-p10-1.osuosl.org + (key from go/go-builders-ssh) * go-le-bionic-1: (30 GB RAM, 50 GB disk, 20 cores, POWER9) Linux go-le-bionic-2 5.4.0-125-generic #141-Ubuntu SMP Wed Aug 10 13:41:48 UTC 2022 ppc64le ppc64le ppc64le GNU/Linux diff --git a/env/linux-ppc64le/osuosl/p10.patch b/env/linux-ppc64le/osuosl/p10.patch new file mode 100644 index 00000000..636a0c55 --- /dev/null +++ b/env/linux-ppc64le/osuosl/p10.patch @@ -0,0 +1,16 @@ +diff --git a/src/internal/buildcfg/cfg.go b/src/internal/buildcfg/cfg.go +index 1cc3d3d..e8553e8 100644 +--- a/src/internal/buildcfg/cfg.go ++++ b/src/internal/buildcfg/cfg.go +@@ -110,8 +110,10 @@ + return 8 + case "power9": + return 9 ++ case "power10": ++ return 10 + } +- Error = fmt.Errorf("invalid GOPPC64: must be power8, power9") ++ Error = fmt.Errorf("invalid GOPPC64: must be power8, power9, power10") + return int(defaultGOPPC64[len("power")] - '0') + } +