From 5701aa0de9e835330aafd18637f05600a7266c12 Mon Sep 17 00:00:00 2001 From: Carlos Amedee Date: Wed, 4 Jan 2023 13:10:17 -0500 Subject: [PATCH] all: remove host-linux-arm64-aws host This change removes the host-linux-arm64-aws change. It has been replaced with the host-linux-arm64-bullseye host which has been running for a couple of weeks. Fixes golang/go#57584 Change-Id: I096c2467236ba48d4bba32d1a649f3491c510898 Reviewed-on: https://go-review.googlesource.com/c/build/+/460555 Reviewed-by: Heschi Kreinick Run-TryBot: Carlos Amedee Reviewed-by: Carlos Amedee TryBot-Result: Gopher Robot --- cmd/buildlet/stage0/stage0.go | 16 +---- cmd/securitybot/main.go | 2 +- dashboard/builders.go | 14 +--- env/linux-arm64/Dockerfile | 25 ------- env/linux-arm64/aws/Dockerfile | 26 -------- env/linux-arm64/aws/Makefile | 25 ------- env/linux-arm64/aws/README.md | 48 -------------- .../aws/packer_image_aws_arm64.json | 58 ----------------- env/linux-arm64/aws/prepare_image.sh | 65 ------------------- env/linux-arm64/aws/rundockerbuildlet.service | 13 ---- env/linux-arm64/build.sh | 10 --- 11 files changed, 4 insertions(+), 298 deletions(-) delete mode 100644 env/linux-arm64/Dockerfile delete mode 100644 env/linux-arm64/aws/Dockerfile delete mode 100644 env/linux-arm64/aws/Makefile delete mode 100644 env/linux-arm64/aws/README.md delete mode 100644 env/linux-arm64/aws/packer_image_aws_arm64.json delete mode 100644 env/linux-arm64/aws/prepare_image.sh delete mode 100644 env/linux-arm64/aws/rundockerbuildlet.service delete mode 100755 env/linux-arm64/build.sh diff --git a/cmd/buildlet/stage0/stage0.go b/cmd/buildlet/stage0/stage0.go index fa72fbe4..4e9f2868 100644 --- a/cmd/buildlet/stage0/stage0.go +++ b/cmd/buildlet/stage0/stage0.go @@ -89,12 +89,7 @@ func main() { if onGCE { break } - switch env := os.Getenv("GO_BUILDER_ENV"); env { - case "host-linux-arm64-aws": - // No special setup. - default: - panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env)) - } + panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", os.Getenv("GO_BUILDER_ENV"))) } if !awaitNetwork() { @@ -146,8 +141,6 @@ func main() { switch buildEnv { case "host-linux-arm-aws": cmd.Args = append(cmd.Args, os.ExpandEnv("--workdir=${WORKDIR}")) - case "host-linux-arm64-aws": - cmd.Args = append(cmd.Args, os.ExpandEnv("--workdir=${WORKDIR}")) case "host-linux-loong64-3a5000": cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...) cmd.Args = append(cmd.Args, os.ExpandEnv("--workdir=${WORKDIR}")) @@ -181,12 +174,7 @@ func main() { if onGCE { break } - switch buildEnv { - case "host-linux-arm64-aws": - // no special configuration - default: - panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env)) - } + panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env)) case "solaris/amd64", "illumos/amd64": hostType := buildEnv cmd.Args = append(cmd.Args, reverseHostTypeArgs(hostType)...) diff --git a/cmd/securitybot/main.go b/cmd/securitybot/main.go index cc8de3d3..a8ff356d 100644 --- a/cmd/securitybot/main.go +++ b/cmd/securitybot/main.go @@ -479,7 +479,7 @@ var firstClassBuilders = []string{ "linux-386-longtest", "linux-amd64-longtest", "linux-arm-aws", - "linux-arm64-aws", + "linux-arm64", "darwin-amd64-12_0", "darwin-arm64-12", diff --git a/dashboard/builders.go b/dashboard/builders.go index 7ee65481..d6628b63 100644 --- a/dashboard/builders.go +++ b/dashboard/builders.go @@ -337,14 +337,6 @@ var Hosts = map[string]*HostConfig{ isEC2: true, SSHUsername: "root", }, - "host-linux-arm64-aws": { - Notes: "Debian Buster, EC2 arm64 instance. See x/build/env/linux-arm64/aws", - VMImage: "ami-03089323a1d38e652", - ContainerImage: "gobuilder-arm64-aws:latest", - machineType: "m6g.xlarge", - isEC2: true, - SSHUsername: "root", - }, "host-linux-arm64-bullseye": { Notes: "Debian Bullseye", ContainerImage: "linux-arm64-bullseye:latest", @@ -2651,10 +2643,6 @@ func init() { return atLeastGo1(goBranch, 20) && buildRepoByDefault(repo) }, }) - addBuilder(BuildConfig{ - Name: "linux-arm64-aws", - HostType: "host-linux-arm64-aws", - }) addBuilder(BuildConfig{ Name: "linux-arm64", HostType: "host-linux-arm64-bullseye", @@ -2663,7 +2651,7 @@ func init() { }) addBuilder(BuildConfig{ Name: "linux-arm64-boringcrypto", - HostType: "host-linux-arm64-aws", + HostType: "host-linux-arm64-bullseye", env: []string{ "GOEXPERIMENT=boringcrypto", "GO_DISABLE_OUTBOUND_NETWORK=1", diff --git a/env/linux-arm64/Dockerfile b/env/linux-arm64/Dockerfile deleted file mode 100644 index 5722d7dc..00000000 --- a/env/linux-arm64/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -FROM debian:buster - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get install --yes \ - gcc curl strace \ - ca-certificates netbase \ - procps lsof psmisc \ - openssh-server - -RUN mkdir /usr/local/go-bootstrap && \ - curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-arm64.tar.gz | \ - tar -C /usr/local/go-bootstrap -zxv - -ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap -RUN curl -o /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-arm64 && \ - chmod +x /usr/local/bin/stage0 - -ENV GO_BUILD_KEY_DELETE_AFTER_READ true -ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey diff --git a/env/linux-arm64/aws/Dockerfile b/env/linux-arm64/aws/Dockerfile deleted file mode 100644 index f81c5f4e..00000000 --- a/env/linux-arm64/aws/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -FROM debian:buster - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && \ - apt-get install --yes \ - gcc curl strace \ - ca-certificates netbase \ - procps lsof psmisc \ - openssh-server - -RUN mkdir /usr/local/go-bootstrap && \ - curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-arm64.tar.gz | \ - tar -C /usr/local/go-bootstrap -zxv - -ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap -RUN curl -o /usr/local/bin/stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-arm64 && \ - chmod +x /usr/local/bin/stage0 - -ENV GO_BUILDER_ENV host-linux-arm64-aws - -CMD ["/usr/local/bin/stage0"] diff --git a/env/linux-arm64/aws/Makefile b/env/linux-arm64/aws/Makefile deleted file mode 100644 index 59a0efe9..00000000 --- a/env/linux-arm64/aws/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2020 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# run on arm64 instance -prod-build: Dockerfile - sudo docker build -t gcr.io/symbolic-datum-552/gobuilder-arm64-aws:latest . - -# run on arm64 instance -prod-push: prod-build - sudo docker login -u oauth2accesstoken https://gcr.io - sudo docker push gcr.io/symbolic-datum-552/gobuilder-arm64-aws:latest - sudo rm /root/.docker/config.json - -env-var-check: -ifndef AWS_ACCESS_KEY_ID - $(error AWS_ACCESS_KEY_ID env var is not set) -endif - -ifndef AWS_SECRET_ACCESS_KEY - $(error AWS_SECRET_ACCESS_KEY env var is not set) -endif - -create-aws-image: env-var-check - packer build packer_image_aws_arm64.json diff --git a/env/linux-arm64/aws/README.md b/env/linux-arm64/aws/README.md deleted file mode 100644 index 934f2af1..00000000 --- a/env/linux-arm64/aws/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# AWS Linux ARM64 Builders - -## Machines - -The AWS builders use the m6 instance types which are arm64 based machines of varying specifications. -The base type used will be m6g.xlarge 4 vCPUs, 16384 MiB. - -## Machine Image - -Machine images are stored on AWS EBS service as a snapshot. New VMs can use the snapshot as an image -by providing the AMI ID as the base image when a new VM is created. The machine image will be configured -to install and initialize rundockerbuildlet. - -### Creating a New Image - -Requirements: - -Two environmental variables are required to be set before initiating the command: -`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` should be set with the appropriate values. - -The [packer](https://www.packer.io) binary should be in `PATH`. - -Command: - -`make create-aws-image` - -or - -`AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= make create-aws-image` - -## Buildlet Container Image - -Buildlet container images must be build on an arm64 instance with the proper credentials. The instructions -are as follows: - -* In your normal gcloud dev environment, retrieve a short-lived access token: - - `you@dev:~$ gcloud auth print-access-token` - -* On an arm64 instance, clone the build repository. - -* cd into the `env/linux-arm64/aws` directory. - -* Execute: `make prod-push` - -* When prompted for your password, paste in the access token from the first step. - -* Ensure `/root/.docker/config.json` has been deleted. diff --git a/env/linux-arm64/aws/packer_image_aws_arm64.json b/env/linux-arm64/aws/packer_image_aws_arm64.json deleted file mode 100644 index 6bfc569a..00000000 --- a/env/linux-arm64/aws/packer_image_aws_arm64.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "region": "us-east-2" - }, - "builders": [ - { - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "ami_name": "go-linux-arm64-{{timestamp}}", - "ami_description": "Image for linux-arm64 Go builder", - "instance_type": "a1.medium", - "region": "{{user `region`}}", - "secret_key": "{{user `aws_secret_key`}}", - "source_ami_filter": { - "filters": { - "architecture": "arm64", - "virtualization-type": "hvm", - "name": "debian-10-arm64-*", - "root-device-type": "ebs" - }, - "owners": ["136693071363"], - "most_recent": true - }, - "decode_authorization_messages": true, - "ssh_username": "admin", - "tags": { - "Name": "Debian", - "Created": "{{isotime \"2006-01-02\"}}", - "OS": "Debian 10 Buster", - "Release": "Latest", - "Base_AMI_Name": "{{ .SourceAMIName }}", - "Extra": "{{ .SourceAMITags.TagName }}", - "Description": "{{user `description`}}" - }, - "launch_block_device_mappings": [ - { - "device_name": "/dev/xvda", - "volume_size": 10, - "volume_type": "gp2", - "delete_on_termination": true - } - ] - } - ], - "provisioners": [ - { - "type": "file", - "source": "./rundockerbuildlet.service", - "destination": "/tmp/rundockerbuildlet.service" - }, - { - "type": "shell", - "script": "./prepare_image.sh" - } - ] -} diff --git a/env/linux-arm64/aws/prepare_image.sh b/env/linux-arm64/aws/prepare_image.sh deleted file mode 100644 index 0f808de2..00000000 --- a/env/linux-arm64/aws/prepare_image.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# -# Installs all dependencies for a Debian 10 linux arm64 Host. -# - -set -euxo pipefail - -TMP_DIR="$(mktemp -d)" -GO_PATH="$TMP_DIR/gopath" - -sudo apt-get update && sudo apt-get upgrade -y - -sudo apt-get install -y \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - gnupg2 \ - jq \ - software-properties-common - -curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add - - -sudo add-apt-repository "deb [arch=arm64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" - -sudo apt-get update - -sudo apt-get install -y \ - docker-ce \ - docker-ce-cli \ - containerd.io - -sudo usermod -aG docker "$USER" - -# retrieve the latest version of Go -GO_VERSION="$(curl -s https://golang.org/dl/?mode=json | jq --raw-output '.[0].version')" -GO_PACKAGE="$GO_VERSION.linux-arm64.tar.gz" -GO_SHA="$(curl -s https://golang.org/dl/?mode=json | jq --raw-output '.[0].files | map(select(.arch == "arm64")) | .[0].sha256')" - -# download Go package -curl -o "$TMP_DIR/$GO_PACKAGE" -L "https://golang.org/dl/$GO_PACKAGE" - -# verify sha256 shasum" -echo "$GO_SHA $TMP_DIR/$GO_PACKAGE" | sha256sum --check --status - -# unzip Go package -tar -xvf "$TMP_DIR/$GO_PACKAGE" -C "$TMP_DIR" - -# build rundockerbuildlet -mkdir -p "$GO_PATH" -GOPATH="$GO_PATH" "$TMP_DIR/go/bin/go" get -u golang.org/x/build/cmd/rundockerbuildlet -GOPATH="$GO_PATH" "$TMP_DIR/go/bin/go" build -o "$TMP_DIR/rundockerbuildlet" golang.org/x/build/cmd/rundockerbuildlet -sudo mv "$TMP_DIR/rundockerbuildlet" /usr/local/bin/rundockerbuildlet - -sudo mv /tmp/rundockerbuildlet.service /etc/systemd/user/rundockerbuildlet.service -sudo systemctl enable /etc/systemd/user/rundockerbuildlet.service -sudo systemctl start rundockerbuildlet - -# remove temporary directory -rm -fr "$TMP_DIR" diff --git a/env/linux-arm64/aws/rundockerbuildlet.service b/env/linux-arm64/aws/rundockerbuildlet.service deleted file mode 100644 index a9229dfd..00000000 --- a/env/linux-arm64/aws/rundockerbuildlet.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Run Buildlets in Docker -After=network.target - -[Install] -WantedBy=network-online.target - -[Service] -Type=simple -RemainAfterExit=true -ExecStart=/usr/local/bin/rundockerbuildlet \ - -env=host-linux-arm64-aws \ - -image=gobuilder-arm64-aws diff --git a/env/linux-arm64/build.sh b/env/linux-arm64/build.sh deleted file mode 100755 index 8fb6017a..00000000 --- a/env/linux-arm64/build.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Copyright 2022 Go Authors All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# -# This is run on the arm64 host, with the Dockerfile in the same directory, -# by the build scripts in linaro and packet subdirectories. - -docker build -t golang.org/linux-arm64 .