all: upgrade scaleway linux-arm builders, stop using deprecated --reverse flag

Updates golang/go#21260 (no more buildlets using the --reverse flag)
Updates golang/go#33574 (linux-arm kernel+userspace updated)

Change-Id: I7455f6fa3e851f1f9f81d6f1eb487ef7e4bea55b
Reviewed-on: https://go-review.googlesource.com/c/build/+/205603
Reviewed-by: Bryan C. Mills <bcmills@google.com>
This commit is contained in:
Brad Fitzpatrick 2019-11-05 17:10:39 +00:00
Родитель 621ef1f246
Коммит d489097173
7 изменённых файлов: 214 добавлений и 61 удалений

Просмотреть файл

@ -4,4 +4,4 @@
# golang.org/x/build/cmd/genbootstrap
The genbootstrap command prepares GO_BOOTSTRAP tarballs suitable for use on builders.
The genbootstrap command prepares GOROOT_BOOTSTRAP tarballs suitable for use on builders.

Просмотреть файл

@ -3,7 +3,7 @@
// license that can be found in the LICENSE file.
/*
The genbootstrap command prepares GO_BOOTSTRAP tarballs suitable for
The genbootstrap command prepares GOROOT_BOOTSTRAP tarballs suitable for
use on builders. It's a wrapper around bootstrap.bash. After
bootstrap.bash produces the full output, genbootstrap trims it up,
removing unnecessary and unwanted files.

Просмотреть файл

@ -22,6 +22,8 @@ import (
"runtime"
"strings"
"time"
"golang.org/x/build/buildenv"
)
var (
@ -32,6 +34,7 @@ var (
keyFile = flag.String("key", "/etc/gobuild.key", "go build key file")
builderEnv = flag.String("env", "", "optional GO_BUILDER_ENV environment variable value to set in the guests")
cpu = flag.Int("cpu", 0, "if non-zero, how many CPUs to assign from the host and pass to docker run --cpuset-cpus")
pull = flag.Bool("pull", false, "whether to pull the the --image before each container starting")
)
var (
@ -42,26 +45,21 @@ var (
func main() {
flag.Parse()
key, err := ioutil.ReadFile(*keyFile)
if err != nil {
log.Fatalf("error reading build key from --key=%s: %v", *keyFile, err)
if onScaleway() {
*memory = ""
*image = "eu.gcr.io/symbolic-datum-552/scaleway-builder"
*pull = true
*numInst = 1
*basename = "scaleway"
initScalewayMeta()
}
buildKey = bytes.TrimSpace(key)
buildKey = getBuildKey()
if *image == "" {
log.Fatalf("docker --image is required")
}
// If we're on linux/arm, see if we're running in a scaleway
// image (which contains the oc-metadata command) and, if so,
// fetch our instance metadata to make the reported hostname match
// the instance hostname.
if runtime.GOOS == "linux" && runtime.GOARCH == "arm" {
if _, err := os.Stat("/usr/local/bin/oc-metadata"); err == nil {
initScalewayMeta()
}
}
log.Printf("Started. Will keep %d copies of %s running.", *numInst, *image)
for {
if err := checkFix(); err != nil {
@ -71,6 +69,34 @@ func main() {
}
}
func onScaleway() bool {
if *builderEnv == "host-linux-arm-scaleway" {
return true
}
if runtime.GOOS == "linux" && runtime.GOARCH == "arm" {
if _, err := os.Stat("/usr/local/bin/oc-metadata"); err == nil {
return true
}
}
return false
}
func getBuildKey() []byte {
key, err := ioutil.ReadFile(*keyFile)
if err != nil {
if onScaleway() {
const prefix = "buildkey_host-linux-arm-scaleway_"
for _, tag := range scalewayMeta.Tags {
if strings.HasPrefix(tag, prefix) {
return []byte(strings.TrimPrefix(tag, prefix))
}
}
}
log.Fatalf("error reading build key from --key=%s: %v", *keyFile, err)
}
return bytes.TrimSpace(key)
}
func checkFix() error {
running := map[string]bool{}
@ -140,6 +166,14 @@ func checkFix() error {
continue
}
if *pull {
log.Printf("Pulling %s ...", *image)
out, err := exec.Command("docker", "pull", *image).CombinedOutput()
if err != nil {
log.Printf("docker pull %s failed: %v, %s", *image, err, out)
}
}
log.Printf("Creating %s ...", name)
keyFile := fmt.Sprintf("/tmp/buildkey%02d/gobuildkey", num)
if err := os.MkdirAll(filepath.Dir(keyFile), 0700); err != nil {
@ -157,12 +191,19 @@ func checkFix() error {
if *memory != "" {
cmd.Args = append(cmd.Args, "--memory="+*memory)
}
if *builderEnv != "" {
cmd.Args = append(cmd.Args, "-e", "GO_BUILDER_ENV="+*builderEnv)
}
if *cpu > 0 {
cmd.Args = append(cmd.Args, fmt.Sprintf("--cpuset-cpus=%d-%d", *cpu*(num-1), *cpu*num-1))
}
if *builderEnv != "" {
cmd.Args = append(cmd.Args, "-e", "GO_BUILDER_ENV="+*builderEnv)
}
if u := buildletBinaryURL(); u != "" {
cmd.Args = append(cmd.Args, "-e", "META_BUILDLET_BINARY_URL="+u)
}
cmd.Args = append(cmd.Args,
"-e", "GO_BUILD_KEY_PATH=/buildkey/gobuildkey",
"-e", "GO_BUILD_KEY_DELETE_AFTER_READ=true",
)
cmd.Args = append(cmd.Args, *image)
out, err := cmd.CombinedOutput()
if err != nil {
@ -180,6 +221,18 @@ type scalewayMetadata struct {
Tags []string `json:"tags"`
}
func (m *scalewayMetadata) HasTag(t string) bool {
if m == nil {
return false
}
for _, v := range m.Tags {
if v == t {
return true
}
}
return false
}
func initScalewayMeta() {
const metaURL = "http://169.254.42.42/conf?format=json"
res, err := http.Get(metaURL)
@ -202,3 +255,15 @@ func removeContainer(container string) {
}
log.Printf("Removed container %s", container)
}
func buildletBinaryURL() string {
if !onScaleway() {
// Only used for Scaleway currently.
return ""
}
env := buildenv.Production
if scalewayMeta.HasTag("staging") {
env = buildenv.Staging
}
return fmt.Sprintf("https://storage.googleapis.com/%s/buildlet.linux-arm", env.BuildletBucket)
}

Просмотреть файл

@ -12,6 +12,7 @@ import (
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
@ -30,7 +31,8 @@ var (
tokenDir = flag.String("token-dir", filepath.Join(os.Getenv("HOME"), "keys"), "directory to read gobuilder-staging.key, gobuilder-master.key and go-scaleway.token from.")
token = flag.String("token", "", "API token. If empty, the file is read from $(token-dir)/go-scaleway.token. Googlers on the Go team can get the value from http://go/golang-scaleway-token")
org = flag.String("org", "1f34701d-668b-441b-bf08-0b13544e99de", "Organization ID (default is bradfitz@golang.org's account)")
image = flag.String("image", "e488d5e3-d278-47a7-8f7d-1154e1f61dc9", "Disk image ID; default is the snapshot we made last")
image = flag.String("image", "13f4c905-3a4b-475a-aaba-a13168e2b6c7", "Disk image ID; default is the snapshot we made last")
bootscript = flag.String("bootscript", "5c8e4527-d166-4844-b6c6-087d7a6f5fb0", "Bootscript ID; empty means to use the default for the image. But our images don't have a correct default.")
num = flag.Int("n", 0, "Number of servers to create; if zero, defaults to a value as a function of --staging")
tags = flag.String("tags", "", "Comma-separated list of tags. The build key tags should be of the form 'buildkey_linux-arm_HEXHEXHEXHEXHEX'. If empty, it's automatic.")
staging = flag.Bool("staging", false, "If true, deploy staging instances (with staging names and tags) instead of prod.")
@ -38,6 +40,7 @@ var (
list = flag.Bool("list", false, "If true, list all prod (or staging, if -staging) servers, including missing ones.")
fixInterval = flag.Duration("fix-interval", 10*time.Minute, "Interval to wait before running again (only applies to daemon mode)")
daemonMode = flag.Bool("daemon", false, "Run in daemon mode in a loop")
ipv6 = flag.Bool("ipv6", false, "enable IPv6 on scaleway instances")
)
const (
@ -144,9 +147,27 @@ func checkServers() {
for i := 1; i <= *num; i++ {
name := serverName(i)
server := servers[name]
if server.Image != nil && server.Image.ID != *image {
log.Printf("server %s, state %q, running wrong image %s (want %s)", name, server.State, server.Image.ID, *image)
switch server.State {
case "running":
log.Printf("powering off %s ...", name)
if err := cl.PowerOff(server.ID); err != nil {
log.Printf("PowerOff(%q (%q)): %v", server.ID, name, err)
}
case "stopped":
log.Printf("deleting %s ...", name)
if err := cl.Delete(server.ID); err != nil {
log.Printf("Delete(%q (%q)): %v", server.ID, name, err)
}
}
}
if server.Connected != nil {
continue
}
if server.State == "running" {
if time.Time(server.ModificationDate).Before(time.Now().Add(15 * time.Minute)) {
log.Printf("rebooting old running-but-disconnected %q server...", name)
@ -171,11 +192,15 @@ func checkServers() {
Image: *image,
CommercialType: ctype,
Tags: tags,
EnableIPV6: *ipv6,
BootType: "bootscript", // the "local" boot mode doesn't work on C1,
Bootscript: *bootscript,
})
if err != nil {
log.Fatal(err)
}
log.Printf("Doing req %q", body)
log.Printf("sending createServerRequest: %s", body)
// TODO: update to their new API path format that includes the zone.
req, err := http.NewRequest("POST", scalewayAPIBase+"/servers", bytes.NewReader(body))
if err != nil {
log.Fatal(err)
@ -186,7 +211,12 @@ func checkServers() {
if err != nil {
log.Fatal(err)
}
log.Printf("Create of %v: %v", i, res.Status)
if res.StatusCode == http.StatusOK {
log.Printf("created %v", i)
} else {
slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
log.Printf("creating number %v, %s: %s", i, res.Status, slurp)
}
res.Body.Close()
}
@ -210,6 +240,9 @@ type createServerRequest struct {
Image string `json:"image"`
CommercialType string `json:"commercial_type"`
Tags []string `json:"tags"`
EnableIPV6 bool `json:"enable_ipv6,omitempty"`
BootType string `json:"boot_type,omitempty"` // local, bootscript, rescue; the default of local doesn't work on C1 machines
Bootscript string `json:"bootscript,omitempty"`
}
type Client struct {
@ -221,7 +254,7 @@ type Client struct {
// This is currently unused. An earlier version of this tool used it briefly before
// changing to use the reboot action. We might want this later.
func (c *Client) Delete(serverID string) error {
req, _ := http.NewRequest("DELETE", scalewayAPIBase+"/servers/"+serverID, nil)
req, _ := http.NewRequest("DELETE", scalewayAPIBase+"/instance/v1/zones/fr-par-1/servers/"+serverID, nil)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Auth-Token", c.Token)
res, err := http.DefaultClient.Do(req)
@ -230,7 +263,8 @@ func (c *Client) Delete(serverID string) error {
}
defer res.Body.Close()
if res.StatusCode != http.StatusNoContent {
return fmt.Errorf("error deleting %s: %v", serverID, res.Status)
slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 1<<10))
return fmt.Errorf("error deleting %s: %v, %s", serverID, res.Status, slurp)
}
return nil
}
@ -239,6 +273,10 @@ func (c *Client) PowerOn(serverID string) error {
return c.serverAction(serverID, "poweron")
}
func (c *Client) PowerOff(serverID string) error {
return c.serverAction(serverID, "poweroff")
}
func (c *Client) serverAction(serverID, action string) error {
req, _ := http.NewRequest("POST", scalewayAPIBase+"/servers/"+serverID+"/action", strings.NewReader(fmt.Sprintf(`{"action":"%s"}`, action)))
req.Header.Set("Content-Type", "application/json")
@ -325,7 +363,9 @@ func defaultBuilderTags(baseKeyFile string) string {
log.Fatal(err)
}
var tags []string
for _, builder := range []string{"linux-arm", "linux-arm-arm5"} {
for _, builder := range []string{
"host-linux-arm-scaleway",
} {
h := hmac.New(md5.New, bytes.TrimSpace(slurp))
h.Write([]byte(builder))
tags = append(tags, fmt.Sprintf("buildkey_%s_%x", builder, h.Sum(nil)))

27
env/linux-arm/scaleway/Dockerfile поставляемый
Просмотреть файл

@ -2,28 +2,23 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
FROM scaleway/ubuntu:armhf-xenial
FROM golang:1.13 AS build
RUN go get golang.org/x/build/cmd/buildlet/stage0
FROM debian:buster
RUN apt-get update
RUN apt-get install --yes \
gcc strace procps psmisc libc6-dev
gcc strace procps psmisc libc6-dev curl netbase \
openssh-server
RUN curl -L -o go1.8.1.tar.gz https://golang.org/dl/go1.8.1.linux-armv6l.tar.gz && \
tar fxzv go1.8.1.tar.gz -C /usr/local
RUN curl -L https://golang.org/dl/go1.13.4.linux-armv6l.tar.gz | \
tar xzv -C /usr/local && /usr/local/go/bin/go version
ENV GO_BOOTSTRAP=/usr/local/go
ENV GOROOT_BOOTSTRAP=/usr/local/go
# compiled stage0 binary must be in working dir
COPY stage0 /usr/local/bin/stage0
COPY --from=build /go/bin/stage0 /usr/local/bin/stage0
ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey
ENV GO_BUILD_KEY_DELETE_AFTER_READ true
ENV GO_BUILDER_ENV host-linux-arm-scaleway
# env specific
ARG buildlet_bucket
ENV META_BUILDLET_BINARY_URL "https://storage.googleapis.com/$buildlet_bucket/buildlet.linux-arm"
CMD ["/usr/local/bin/stage0"]
CMD ["/usr/local/bin/stage0"]

89
env/linux-arm/scaleway/README поставляемый
Просмотреть файл

@ -1,30 +1,83 @@
C1 image created with docker base image using
`armv7l 4.10.8 docker #1` bootscript.
Machines:
C1 machines:
$ ssh -i ~/keys/id_ed25519_golang1 root@$C1_SERVER_IP
(key: http://go/go-builders-ssh)
Machine setup:
# local:
$ scp env/linux-arm/scaleway/* root@$C1_SERVER_IP:~/scaleway
$ GOARCH=arm GOOS=linux go install golang.org/x/build/cmd/rundockerbuildlet && \
scp ~/bin/linux_arm/rundockerbuildlet root@$C1_SERVER_IP:/usr/local/bin
$ GOARCH=arm GOOS=linux go install golang.org/x/build/cmd/buildlet/stage0 && \
scp ~/bin/linux_arm/stage0 root@$C1_SERVER_IP:~/scaleway
# NOTE: commit SHA of rundockerbuildlet/stage0 on instance snapshot (06/19/17)
# golang.org/x/build is 282d813.
* start with Ubuntu bionic
* install docker-ce from Docker
* GOOS=linux GOARCH=arm go install golang.org/x/build/cmd/rundockerbuildlet
* scp -C -i ~/keys/id_ed25519_golang1 $(GOOS=linux GOARCH=arm go list -f {{.Target}} golang.org/x/build/cmd/rundockerbuildlet) root@$C1_SERVER_IP:/usr/local/bin
* scp -i ~/keys/id_ed25519_golang1 rundockerbuildlet.service root@$C1_SERVER_IP:/etc/systemd/user/
* ssh -i ~/keys/id_ed25519_golang1 root@$C1_SERVER_IP systemctl enable /etc/systemd/user/rundockerbuildlet.service
* ssh -i ~/keys/id_ed25519_golang1 root@$C1_SERVER_IP systemctl start rundockerbuildlet
$ scp rundockerbuildlet.service root@$C1_SERVER_IP:/etc/systemd/user/
* stop it
* snapshot its attached volume, name it (e.g. "buildlet-2019-11-05-1"), note its UUID
* click "Images" in top tab, create an image from that same snapshot. Can use the same name. (different namespaces)
* click "..." and "More info" on that image, get its ID (e.g. "13f4c905-3a4b-475a-aaba-a13168e2b6c7")
* copy that to cmd/scaleway's default flag value
* also update cmd/scaleway's bootscript value, if needed:
# c1 server:
First, find out which bootscript (netboot kernel, initrd, etc) that the original image was using:
$ echo "<BUILDER KEY>" > /etc/gobuild.key > /root/.gobuildkey
bradfitz@go:~$ scw images --no-trunc | grep Bionic
Ubuntu_Bionic_ML_10_1 latest 47d58f71-8382-48d1-88cd-75e5f1ed7df6 8 months [ par1] [x86_64]
Ubuntu_Bionic_ML_9_2 latest 905865bf-e34c-46b0-b7e2-5e11922e6511 8 months [ par1] [x86_64]
Ubuntu_Bionic latest e640c621-305b-45f5-975f-a3f80c1cec66 18 months [ams1 par1] [arm arm64 x86_64]
bradfitz@go:~$ scw inspect e640c621-305b-45f5-975f-a3f80c1cec66
IMAGEID FROM NAME ZONE ARCH
- f974feac image:ubuntu-bionic Ubuntu Bionic par1 x86_64
- f63fe42a image:ubuntu-bionic Ubuntu Bionic par1 arm
- b4bdbee1 image:ubuntu-bionic Ubuntu Bionic par1 arm64
FATA[0000] Too many candidates for e640c621-305b-45f5-975f-a3f80c1cec66 (3)
bradfitz@go:~$ scw inspect f63fe42a
[{
"id": "f63fe42a-900f-4a5e-ba99-ab0e59469b7e",
"name": "Ubuntu Bionic Beaver",
"creation_date": "2019-03-05T10:27:18.195593+00:00",
"modification_date": "2019-03-05T13:32:45.036625+00:00",
"root_volume": {
"id": "81992ee7-89b5-44f7-a260-5b4594c74b86",
"size": 10000000000,
"name": "snapshot-664f5068-f21e-44ba-8a99-ea2dfd6fd91f-2019-03-05_10:27",
"volume_type": "l_ssd"
},
"public": true,
"default_bootscript": {
"bootcmdargs": "LINUX_COMMON scaleway boot=local nbd.max_part=16",
"dtb": "dtb/c1-armv7l-mainline-lts-4.9-4.9.93-rev1",
"initrd": "initrd/uInitrd-Linux-armv7l-v3.14.6",
"kernel": "kernel/armv7l-mainline-lts-4.9-4.9.93-rev1",
"architecture": "arm",
"id": "5c8e4527-d166-4844-b6c6-087d7a6f5fb0",
"organization": "11111111-1111-4111-8111-111111111111",
"title": "armv7l mainline 4.9.93 rev1"
},
"organization": "51b656e3-4865-41e8-adbc-0c45bdd780db",
"arch": "arm"
}]
$ apt-get remove --purge unattended-upgrades
Look use that default_bootscript.id value as cmd/scaleway's --bootscript flag.
$ systemctl enable /etc/systemd/user/rundockerbuildlet.service
$ systemctl start rundockerbuildlet.service
Updating the docker image on eu.gcr.io:
* create a dev Ubuntu image on scaleway
* install Docker on it
* copy Dockerfile to it
root@scaleway:~# cat Dockerfile | docker build -t eu.gcr.io/symbolic-datum-552/scaleway-builder -
* in your normal gcloud dev environment, get a short-lived access token:
you@dev:~$ gcloud auth print-access-token
* copy/paste it
* back on scaleway, log in, and paste the token when it asks for the password:
root@scaleway:~# docker login -u oauth2accesstoken https://eu.gcr.io
Password: <paste>
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
root@scaleway:~# docker push eu.gcr.io/symbolic-datum-552/scaleway-builder

Просмотреть файл

@ -7,7 +7,7 @@ WantedBy=network-online.target
[Service]
Type=simple
ExecStart=/usr/local/bin/rundockerbuildlet -basename=scaleway -image=gobuilder-arm-scaleway:latest -n=1
ExecStart=/usr/local/bin/rundockerbuildlet -env=host-linux-arm-scaleway
Restart=always
RestartSec=2
StartLimitInterval=0
StartLimitInterval=0