cmd/makemac: add full instance management to makemac

Currently makemac is extremely minimal, all it does is renew existing
leases.  It does not attempt to detect broken leases or create new
leases. Over time as leases disappear for various reasons, the pool
slowly dwindles, and a human must come along and add new leases.

Extend makemac to perform complete lifecycle management. config.go
specifies the desired count of each image type, and makemac attempts
to maintain that many healthy leases.

There are several different ways that a lease may be unhealthy:

It may fail initial boot. If it fails to connect to the hypervisor,
MacService will automatically remove it eventually. If it connects to
the hypervisor, but not to LUCI, then it will appear healthy in
MacService but be missing from swarming.

It may succeed initial boot and successfully connect to LUCI, but
eventually freeze, crash, etc. This case will appears as a "dead" bot
on LUCI, and may or may not be automatically removed from MacService
depending on the nature of the freeze/crash.

makemac attempts to detect and handle all of these cases. For example,
if LUCI reports a bot as "dead", but MacService still reports it as
alive, makemac will destroy the lease.

Since makemac can now perform destructive actions, we need to add a
bit more safety. Leases created by makemac will set the MacService
lease "project name" to "makemac". The "project name" is effectively
just a tag on the lease.  makemac will only operate on leases with the
"makemac" project. All other leases (such as those manually created by
a human) will be left alone.

Image updates can be performed by changing the image SHA in config.go.
handleObsoleteLeases will automatically destroy old leases using the
old image on the next run.

Change-Id: I9bc53cb5812784adbb5cacf9fb224d64d063c089
Reviewed-on: https://go-review.googlesource.com/c/build/+/562399
Auto-Submit: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>

cmd/makemac/config.go

@@ -0,0 +1,70 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"log"
+)
+
+// imageConfig describes how many instances of a specific image type should
+// exist.
+type imageConfig struct {
+	Name     string // short image name
+	Image    string // image SHA
+	MinCount int    // minimum instance count to maintain
+}
+
+// Production image configuration.
+//
+// After changing an image here, makemac will automatically destroy instances
+// with the old image.
+var prodImageConfig = []imageConfig{
+	{
+		Name:     "darwin-amd64-11",
+		Image:    "f0cc898922b37726f6d5ad7b260e92b0443c6289b535cb0a32fd2955abe8adcc",
+		MinCount: 10,
+	},
+	{
+		Name:     "darwin-amd64-12",
+		Image:    "0a45171fb12a7efc3e7c5170b3292e592822dfc63c15aca0d093d94621097b8d",
+		MinCount: 10,
+	},
+	{
+		Name:     "darwin-amd64-13",
+		Image:    "f1bda73984f0725f2fa147d277ef87498bdec170030e1c477ee3576b820f1fb6",
+		MinCount: 10,
+	},
+	{
+		Name:     "darwin-amd64-14",
+		Image:    "ad1a56b7fec85ead9992b04444c4b5aef81becf38f85529976646f14a9ce5410",
+		MinCount: 10,
+	},
+}
+
+// imageConfigMap returns a map from imageConfig.Image to imageConfig.
+func imageConfigMap(cc []imageConfig) map[string]*imageConfig {
+	m := make(map[string]*imageConfig)
+	for _, c := range cc {
+		c := c
+		if _, ok := m[c.Image]; ok {
+			panic(fmt.Sprintf("duplicate image %s in image config", c.Image))
+		}
+		m[c.Image] = &c
+	}
+	return m
+}
+
+func init() {
+	// Panic if prodImageConfig contains duplicates.
+	imageConfigMap(prodImageConfig)
+}
+
+func logImageConfig(cc []imageConfig) {
+	log.Printf("Image configuration:")
+	for _, c := range cc {
+		log.Printf("\t%s: image=%s\tcount=%d", c.Name, c.Image, c.MinCount)
+	}
+}

cmd/makemac/deployment-prod.yaml

@@ -21,7 +21,7 @@ spec:
       - name: makemac
         image: gcr.io/symbolic-datum-552/makemac:latest
         imagePullPolicy: Always
-        command: ["/makemac", "-api-key=secret:macservice-api-key"]
+        command: ["/makemac", "-macservice-api-key=secret:macservice-api-key"]
         resources:
           requests:
             cpu: "1"

cmd/makemac/macservice.go

@@ -0,0 +1,46 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"log"
+
+	"golang.org/x/build/internal/macservice"
+)
+
+// Interface matching macservice.Client to use for test mocking.
+type macServiceClient interface {
+	Lease(macservice.LeaseRequest) (macservice.LeaseResponse, error)
+	Renew(macservice.RenewRequest) (macservice.RenewResponse, error)
+	Vacate(macservice.VacateRequest) error
+	Find(macservice.FindRequest) (macservice.FindResponse, error)
+}
+
+// readOnlyMacServiceClient wraps a macServiceClient, logging instead of
+// performing mutating actions. Used for dry run mode.
+type readOnlyMacServiceClient struct {
+	mc macServiceClient
+}
+
+func (r readOnlyMacServiceClient) Lease(req macservice.LeaseRequest) (macservice.LeaseResponse, error) {
+	log.Printf("DRY RUN: Create lease with image %s", req.InstanceSpecification.DiskSelection.ImageHashes.BootSHA256)
+	return macservice.LeaseResponse{
+		PendingLease: macservice.Lease{LeaseID: "dry-run-lease"},
+	}, nil
+}
+
+func (r readOnlyMacServiceClient) Renew(req macservice.RenewRequest) (macservice.RenewResponse, error) {
+	log.Printf("DRY RUN: Renew lease %s with duration %s", req.LeaseID, req.Duration)
+	return macservice.RenewResponse{}, nil // Perhaps fake RenewResponse.Expires?
+}
+
+func (r readOnlyMacServiceClient) Vacate(req macservice.VacateRequest) error {
+	log.Printf("DRY RUN: Vacate lease %s", req.LeaseID)
+	return nil
+}
+
+func (r readOnlyMacServiceClient) Find(req macservice.FindRequest) (macservice.FindResponse, error) {
+	return r.mc.Find(req)
+}

cmd/makemac/main.go

@@ -2,81 +2,507 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Command makemac ensures that MacService instances continue running.
+// Command makemac manages MacService instances for LUCI.
-// Currently, it simply renews any existing leases.
+//
+// It performs several different operations:
+//
+// * Detects MacService leases that MacService thinks are running, but never
+//   connected to LUCI (failed to boot?) and destroys them.
+// * Detects MacService leases that MacService thinks are running, but LUCI
+//   thinks are dead (froze/crashed?) and destoys them.
+// * Renews MacService leases that both MacService and LUCI agree are healthy
+//   to ensure they don't expire.
+// * Destroys MacService leases with images that are not requested by the
+//   configuration in config.go.
+// * Launches new MacService leases to ensure that there are the at least as
+//   many leases of each type as specified in the configuration in config.go.
 package main
 
 import (
 	"context"
 	"flag"
+	"fmt"
 	"log"
+	"regexp"
+	"sort"
 	"time"
 
+	"go.chromium.org/luci/swarming/client/swarming"
+	spb "go.chromium.org/luci/swarming/proto/api_v2"
 	"golang.org/x/build/internal/macservice"
 	"golang.org/x/build/internal/secret"
+	"golang.org/x/oauth2/google"
 )
 
 var (
-	apiKey = secret.Flag("api-key", "MacService API key")
+	apiKey = secret.Flag("macservice-api-key", "MacService API key")
+	period = flag.Duration("period", 1*time.Hour, "How often to check bots and leases. As a special case, -period=0 checks exactly once and then exits")
 	dryRun = flag.Bool("dry-run", false, "Print the actions that would be taken without actually performing them")
-	period = flag.Duration("period", 2*time.Hour, "How often to check leases. As a special case, -period=0 checks exactly once and then exits")
 )
 
-const renewDuration = "86400s" // 24h
+const (
+	createExpirationDuration = 24*time.Hour
+	createExpirationDurationString = "86400s"
+
+	// Shorter renew expiration is a workaround to detect newly-created
+	// leases. See comment in handleMissingBots.
+	renewExpirationDuration = 23*time.Hour
+	renewExpirationDurationString = "82800s" // 23h
+)
+
+const (
+	swarmingService = "https://chromium-swarm.appspot.com"
+	swarmingPool    = "luci.golang.shared-workers"
+)
+
+const (
+	macServiceCustomer = "golang"
+
+	// Leases managed by makemac have ProjectName "makemac". Leases without
+	// this project will not be touched.
+	managedProject = "makemac"
+)
 
 func main() {
 	secret.InitFlagSupport(context.Background())
 	flag.Parse()
 
-	c := macservice.NewClient(*apiKey)
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
 
-	// Always check once at startup.
+func run() error {
-	checkAndRenewLeases(c)
+	ctx := context.Background()
+
+	var mc macServiceClient
+	mc = macservice.NewClient(*apiKey)
+	if *dryRun {
+		mc = readOnlyMacServiceClient{mc: mc}
+	}
+
+	// Use service account / application default credentials for swarming
+	// authentication.
+	ac, err := google.DefaultClient(ctx)
+	if err != nil {
+		return fmt.Errorf("error creating authenticated client: %w", err)
+	}
+
+	sc, err := swarming.NewClient(ctx, swarming.ClientOptions{
+		ServiceURL:          swarmingService,
+		AuthenticatedClient: ac,
+	})
+	if err != nil {
+		return fmt.Errorf("error creating swarming client: %w", err)
+	}
+
+	logImageConfig(prodImageConfig)
+
+	// Always run once at startup.
+	runOnce(ctx, sc, mc)
 
 	if *period == 0 {
 		// User only wants a single check. We're done.
-		return
+		return nil
 	}
 
 	t := time.NewTicker(*period)
 	for range t.C {
-		checkAndRenewLeases(c)
+		runOnce(ctx, sc, mc)
+	}
+
+	return nil
+}
+
+func runOnce(ctx context.Context, sc swarming.Client, mc macServiceClient) {
+	bots, err := swarmingBots(ctx, sc)
+	if err != nil {
+		log.Printf("Error looking up swarming bots: %v", err)
+		return
+	}
+
+	leases, err := macServiceLeases(mc)
+	if err != nil {
+		log.Printf("Error looking up MacService leases: %v", err)
+		return
+	}
+
+	logSummary(bots, leases)
+
+	// These directly correspond to the operation described in the package
+	// comment above.
+	handleMissingBots(mc, bots, leases)
+	handleDeadBots(mc, bots, leases)
+	renewLeases(mc, leases)
+	handleObsoleteLeases(mc, prodImageConfig, leases)
+	addNewLeases(mc, prodImageConfig, leases)
+}
+
+func leaseIsManaged(l macservice.Lease) bool {
+	return l.VMResourceNamespace.ProjectName == managedProject
+}
+
+func logSummary(bots map[string]*spb.BotInfo, leases map[string]macservice.Instance) {
+	keys := make([]string, 0, len(bots))
+	for k := range bots {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	log.Printf("Swarming bots:")
+	for _, k := range keys {
+		b := bots[k]
+
+		alive := true
+		if b.GetIsDead() {
+			alive = false
+		}
+
+		os := "<unknown OS version>"
+		dimensions := b.GetDimensions()
+		for _, d := range dimensions {
+			if d.Key != "os" {
+				continue
+			}
+			if len(d.Value) == 0 {
+				continue
+			}
+			os = d.Value[len(d.Value)-1] // most specific value last.
+		}
+
+		log.Printf("\t%s: alive=%t\tos=%s", k, alive, os)
+	}
+
+	keys = make([]string, 0, len(leases))
+	for k := range leases {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	log.Printf("MacService leases:")
+	for _, k := range keys {
+		inst := leases[k]
+
+		managed := false
+		if leaseIsManaged(inst.Lease) {
+			managed = true
+		}
+
+		image := inst.InstanceSpecification.DiskSelection.ImageHashes.BootSHA256
+
+		log.Printf("\t%s: managed=%t\timage=%s", k, managed, image)
 	}
 }
 
-func checkAndRenewLeases(c *macservice.Client) {
+// e.g., darwin-amd64-11--39b47cf6-2aaa-4c80-b9cb-b800844fb104.golang.c3.macservice.goog
-	log.Printf("Renewing leases...")
+var botIDRe = regexp.MustCompile(`.*--([0-9a-f-]+)\.golang\..*\.macservice.goog$`)
 
-	resp, err := c.Find(macservice.FindRequest{
+// swarmingBots returns set of bots backed by MacService, as seen by swarming.
+// The map key is the MacService lease ID.
+// Bots may be dead.
+func swarmingBots(ctx context.Context, sc swarming.Client) (map[string]*spb.BotInfo, error) {
+	dimensions := []*spb.StringPair{
+		{
+			Key:   "pool",
+			Value: swarmingPool,
+		},
+		{
+			Key:   "os",
+			Value: "Mac",
+		},
+	}
+	bb, err := sc.ListBots(ctx, dimensions)
+	if err != nil {
+		return nil, fmt.Errorf("error listing bots: %w", err)
+	}
+
+	m := make(map[string]*spb.BotInfo)
+
+	for _, b := range bb {
+		id := b.GetBotId()
+		match := botIDRe.FindStringSubmatch(id)
+		if match == nil {
+			log.Printf("Swarming bot %s is not a MacService bot, skipping...", id)
+			continue
+		}
+
+		lease := match[1]
+		m[lease] = b
+	}
+
+	return m, nil
+}
+
+// macServiceLeases returns the set of active MacService leases.
+func macServiceLeases(mc macServiceClient) (map[string]macservice.Instance, error) {
+	resp, err := mc.Find(macservice.FindRequest{
 		VMResourceNamespace: macservice.Namespace{
 			CustomerName: "golang",
 		},
 	})
 	if err != nil {
-		log.Printf("Error finding leases: %v", err)
+		return nil, fmt.Errorf("error finding leases: %v", err)
-		return
 	}
 
-	if len(resp.Instances) == 0 {
+	m := make(map[string]macservice.Instance)
-		log.Printf("No leases found")
-		return
-	}
 
 	for _, i := range resp.Instances {
-		log.Printf("Renewing lease ID: %s; currently expires: %v...", i.Lease.LeaseID, i.Lease.Expires)
+		m[i.Lease.LeaseID] = i
-		if *dryRun {
+	}
+
+	return m, nil
+}
+
+// handleMissingBots detects MacService leases that MacService thinks are
+// running, but never connected to LUCI (i.e., missing completely from LUCI)
+// and destroys them.
+//
+// These are bots that perhaps never successfully booted?
+func handleMissingBots(mc macServiceClient, bots map[string]*spb.BotInfo, leases map[string]macservice.Instance) {
+	log.Printf("Checking for missing bots...")
+
+	var missing []string
+	for id := range leases {
+		if _, ok := bots[id]; !ok {
+			missing = append(missing, id)
+		}
+	}
+	// Sort to make the logs easier to follow when comparing vs a bot/lease
+	// list.
+	sort.Strings(missing)
+
+	for _, id := range missing {
+		lease := leases[id]
+
+		if !leaseIsManaged(lease.Lease) {
+			log.Printf("Lease %s missing from LUCI, but not managed by makemac; skipping", id)
 			continue
 		}
 
-		rr, err := c.Renew(macservice.RenewRequest{
+		// There is a race window here: if this lease was created in
-			LeaseID:  i.Lease.LeaseID,
+		// the last few minutes, the initial boot may still be ongoing,
-			Duration: renewDuration,
+		// and thus being missing from LUCI is expected. We don't want
+		// to destroy these leases.
+		//
+		// Unfortunately MacService doesn't report lease creation time,
+		// so we can't trivially check for this case. It does report
+		// expiration time. As a workaround, we create new leases with
+		// a 24h expiration time, but renew leases with a 23h
+		// expiration. Thus if we see expiration is >23h from now then
+		// this lease must have been created in the last hour.
+		untilExpiration := time.Until(lease.Lease.Expires)
+		if untilExpiration > renewExpirationDuration {
+			log.Printf("Lease %s missing from LUCI, but created in the last hour (still booting?); skipping", id)
+			continue
+		}
+
+		log.Printf("Lease %s missing from LUCI; failed initial boot?", id)
+		log.Printf("Vacating lease %s...", id)
+		if err := mc.Vacate(macservice.VacateRequest{LeaseID: id}); err != nil {
+			log.Printf("Error vacating lease %s: %v", id, err)
+			continue
+		}
+		delete(leases, id) // Drop from map so future calls know it is gone.
+	}
+}
+
+// handleDeadBots detects MacService leases that MacService thinks are running,
+// but LUCI thinks are dead (froze/crashed?) and destoys them.
+//
+// These are bots that perhaps froze/crashed at some point after starting.
+func handleDeadBots(mc macServiceClient, bots map[string]*spb.BotInfo, leases map[string]macservice.Instance) {
+	log.Printf("Checking for dead bots...")
+
+	var dead []string
+	for id, b := range bots {
+		if b.GetIsDead() {
+			dead = append(dead, id)
+		}
+	}
+	// Sort to make the logs easier to follow when comparing vs a bot/lease
+	// list.
+	sort.Strings(dead)
+
+	for _, id := range dead {
+		lease, ok := leases[id]
+		if !ok {
+			// Dead bot already gone from MacService; nothing to do.
+			continue
+		}
+
+		if !leaseIsManaged(lease.Lease) {
+			log.Printf("Lease %s is dead on LUCI, but still present on MacService, but not managed by makemac; skipping", id)
+			continue
+		}
+
+		// No need to check for newly created leases like we do in
+		// handleMissingBots. If a bot appears as dead on LUCI then it
+		// must have successfully connected at some point.
+
+		log.Printf("Lease %s is dead on LUCI, but still present on MacService; VM froze/crashed?", id)
+		log.Printf("Vacating lease %s...", id)
+		if err := mc.Vacate(macservice.VacateRequest{LeaseID: id}); err != nil {
+			log.Printf("Error vacating lease %s: %v", id, err)
+			continue
+		}
+		delete(leases, id) // Drop from map so future calls know it is gone.
+	}
+}
+
+// renewLeases renews lease expiration on all makemac-managed leases. Note that
+// this may renew leases that will later be removed because their image is no
+// longer required. This is harmless.
+func renewLeases(mc macServiceClient, leases map[string]macservice.Instance) {
+	log.Printf("Renewing leases...")
+
+	var ids []string
+	for id := range leases {
+		ids = append(ids, id)
+	}
+	// Sort to make the logs easier to follow when comparing vs a bot/lease
+	// list.
+	sort.Strings(ids)
+
+	for _, id := range ids {
+		lease := leases[id]
+
+		if !leaseIsManaged(lease.Lease) {
+			log.Printf("Lease %s is not managed by makemac; skipping renew", id)
+			continue
+		}
+
+		// Extra spaces to make expiration line up with the renewal message below.
+		log.Printf("Lease ID: %s currently expires:    %v", lease.Lease.LeaseID, lease.Lease.Expires)
+
+		// Newly created leases have a longer expiration duration than
+		// our renewal expiration duration. Don't renew these, which
+		// would would unintentionally shorten their expiration. See
+		// comment in handleMissingBots.
+		until := time.Until(lease.Lease.Expires)
+		if until > renewExpirationDuration {
+			log.Printf("Lease ID: %s skip renew, current expiration further out than renew expiration", lease.Lease.LeaseID)
+			continue
+		}
+
+		rr, err := mc.Renew(macservice.RenewRequest{
+			LeaseID:  lease.Lease.LeaseID,
+			Duration: renewExpirationDurationString,
 		})
 		if err == nil {
-			// Extra spaces to make fields line up with the message above.
+			log.Printf("Lease ID: %s renewed, now expires: %v", lease.Lease.LeaseID, rr.Expires)
-			log.Printf("Renewed  lease ID: %s; now       expires: %v", i.Lease.LeaseID, rr.Expires)
 		} else {
-			log.Printf("Error renewing lease ID: %s: %v", i.Lease.LeaseID, err)
+			log.Printf("Lease ID: %s error renewing %v", lease.Lease.LeaseID, err)
+		}
+	}
+}
+
+// handleObsoleteLeases vacates any makemac-managed leases with images that are
+// not requested by imageConfigs. This typically occurs when updating makemac
+// to roll out a new image version.
+func handleObsoleteLeases(mc macServiceClient, config []imageConfig, leases map[string]macservice.Instance) {
+	log.Printf("Checking for leases with obsolete images...")
+
+	configMap := imageConfigMap(config)
+
+	var ids []string
+	for id := range leases {
+		ids = append(ids, id)
+	}
+	// Sort to make the logs easier to follow when comparing vs a bot/lease
+	// list.
+	sort.Strings(ids)
+
+	for _, id := range ids {
+		lease := leases[id]
+
+		if !leaseIsManaged(lease.Lease) {
+			log.Printf("Lease %s is not managed by makemac; skipping image check", id)
+			continue
+		}
+
+		image := lease.InstanceSpecification.DiskSelection.ImageHashes.BootSHA256
+		if _, ok := configMap[image]; ok {
+			continue
+		}
+
+		// Config doesn't want instances with this image. Vacate.
+		log.Printf("Lease %s uses obsolete image %s", id, image)
+		log.Printf("Vacating lease %s...", id)
+		if err := mc.Vacate(macservice.VacateRequest{LeaseID: id}); err != nil {
+			log.Printf("Error vacating lease %s: %v", id, err)
+			continue
+		}
+		delete(leases, id) // Drop from map so future calls know it is gone.
+	}
+}
+
+func makeLeaseRequest(image string) macservice.LeaseRequest {
+	return macservice.LeaseRequest{
+		VMResourceNamespace: macservice.Namespace{
+			CustomerName: macServiceCustomer,
+			ProjectName:  managedProject,
+		},
+		InstanceSpecification: macservice.InstanceSpecification{
+			OSType: macservice.MAC,
+			Profile: macservice.V1_MEDIUM_VM,
+			AccessLevel: macservice.GOLANG_OSS,
+			DiskSelection: macservice.DiskSelection{
+				ImageHashes: macservice.ImageHashes{
+					BootSHA256: image,
+				},
+			},
+		},
+		Duration: createExpirationDurationString,
+	}
+}
+
+// addNewLeases adds new MacService leases as needed to ensure that there are
+// at least MinCount makemac-managed leases of each configured image type.
+func addNewLeases(mc macServiceClient, config []imageConfig, leases map[string]macservice.Instance) {
+	log.Printf("Checking if new leases are required...")
+
+	configMap := imageConfigMap(config)
+
+	imageCount := make(map[string]int)
+
+	for _, lease := range leases {
+		if !leaseIsManaged(lease.Lease) {
+			// Don't count leases we don't manage.
+			continue
+		}
+
+		image := lease.InstanceSpecification.DiskSelection.ImageHashes.BootSHA256
+		imageCount[image]++
+	}
+
+	var images []string
+	for image := range configMap {
+		images = append(images, image)
+	}
+	sort.Strings(images)
+
+	log.Printf("Current image lease count:")
+	for _, image := range images {
+		config := configMap[image]
+		gotCount := imageCount[config.Image]
+		log.Printf("\t%s: have %d leases\twant %d leases", config.Image, gotCount, config.MinCount)
+	}
+
+	for _, image := range images {
+		config := configMap[image]
+		gotCount := imageCount[config.Image]
+		need := config.MinCount - gotCount
+		if need <= 0 {
+			continue
+		}
+
+		log.Printf("Image %s: creating %d new leases", config.Image, need)
+		for i := 0; i < need; i++ {
+			log.Printf("Image %s: creating lease %d...", config.Image, i)
+			resp, err := mc.Lease(makeLeaseRequest(config.Image))
+			if err != nil {
+				log.Printf("Image %s: creating lease %d: error %v", config.Image, i, err)
+				continue
+			}
+			log.Printf("Image %s: created lease %s", config.Image, resp.PendingLease.LeaseID)
 		}
 	}
 }

cmd/makemac/main_test.go

@@ -0,0 +1,330 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	spb "go.chromium.org/luci/swarming/proto/api_v2"
+	"golang.org/x/build/internal/macservice"
+)
+
+// recordMacServiceClient is a macserviceClient that records mutating requests.
+type recordMacServiceClient struct {
+	lease  []macservice.LeaseRequest
+	renew  []macservice.RenewRequest
+	vacate []macservice.VacateRequest
+}
+
+func (r *recordMacServiceClient) Lease(req macservice.LeaseRequest) (macservice.LeaseResponse, error) {
+	r.lease = append(r.lease, req)
+	return macservice.LeaseResponse{}, nil // Perhaps fake LeaseResponse.PendingLease.LeaseID?
+}
+
+func (r *recordMacServiceClient) Renew(req macservice.RenewRequest) (macservice.RenewResponse, error) {
+	r.renew = append(r.renew, req)
+	return macservice.RenewResponse{}, nil // Perhaps fake RenewResponse.Expires?
+}
+
+func (r *recordMacServiceClient) Vacate(req macservice.VacateRequest) error {
+	r.vacate = append(r.vacate, req)
+	return nil
+}
+
+func (r *recordMacServiceClient) Find(req macservice.FindRequest) (macservice.FindResponse, error) {
+	return macservice.FindResponse{}, fmt.Errorf("unimplemented")
+}
+
+func TestHandleMissingBots(t *testing.T) {
+	// Test leases:
+	// * "healthy" connected to LUCI, and is healthy.
+	// * "dead" connected to LUCI, but later died.
+	// * "newBooting" never connected to LUCI, but was just created 5min ago.
+	// * "neverBooted" never connected to LUCI, and was created 5hr ago.
+	// * "neverBootedUnmanaged" never connected to LUCI, and was created 5hr ago, but is not managed by makemac.
+	//
+	// handleMissingBots should vacate neverBooted and none of the others.
+	bots := map[string]*spb.BotInfo{
+		"healthy": {BotId: "healthy"},
+		"dead":    {BotId: "dead", IsDead: true},
+	}
+	leases := map[string]macservice.Instance{
+		"healthy": {
+			Lease: macservice.Lease{
+				LeaseID:             "healthy",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 2*time.Hour),
+			},
+		},
+		"newBooting": {
+			Lease: macservice.Lease{
+				LeaseID:             "newBooting",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 5*time.Minute),
+			},
+		},
+		"neverBooted": {
+			Lease: macservice.Lease{
+				LeaseID:             "neverBooted",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 2*time.Hour),
+			},
+		},
+		"neverBootedUnmanaged": {
+			Lease: macservice.Lease{
+				LeaseID:             "neverBootedUnmanaged",
+				VMResourceNamespace: macservice.Namespace{ProjectName: "other"},
+				Expires:             time.Now().Add(createExpirationDuration - 2*time.Hour),
+			},
+		},
+	}
+
+	var mc recordMacServiceClient
+	handleMissingBots(&mc, bots, leases)
+
+	got := mc.vacate
+	want := []macservice.VacateRequest{{LeaseID: "neverBooted"}}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Vacated leases mismatch (-want +got):\n%s", diff)
+	}
+
+	if _, ok := leases["neverBooted"]; ok {
+		t.Errorf("neverBooted present in leases, want deleted")
+	}
+}
+
+func TestHandleDeadBots(t *testing.T) {
+	// Test leases:
+	// * "healthy" connected to LUCI, and is healthy.
+	// * "dead" connected to LUCI, but later died, and the lease is gone from MacService.
+	// * "deadLeasePresent" connected to LUCI, but later died, and the lease is still present on MacService.
+	// * "deadLeasePresentUnmanaged" connected to LUCI, but later died, and the lease is still present on MacService, but is not managed by makemac.
+	// * "neverBooted" never connected to LUCI, and was created 5hr ago.
+	//
+	// handleDeadBots should vacate deadLeasePresent and none of the others.
+	bots := map[string]*spb.BotInfo{
+		"healthy":          {BotId: "healthy"},
+		"dead":             {BotId: "dead", IsDead: true},
+		"deadLeasePresent": {BotId: "deadLeasePresent", IsDead: true},
+	}
+	leases := map[string]macservice.Instance{
+		"healthy": {
+			Lease: macservice.Lease{
+				LeaseID:             "healthy",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 2*time.Hour),
+			},
+		},
+		"deadLeasePresent": {
+			Lease: macservice.Lease{
+				LeaseID:             "deadLeasePresent",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				// Lease created 5 minutes ago. Doesn't matter;
+				// new lease checked don't apply here. See
+				// comment in handleDeadBots.
+				Expires: time.Now().Add(createExpirationDuration - 5*time.Minute),
+			},
+		},
+		"deadLeasePresentUnmanaged": {
+			Lease: macservice.Lease{
+				LeaseID:             "deadLeasePresentUnmanaged",
+				VMResourceNamespace: macservice.Namespace{ProjectName: "other"},
+				Expires:             time.Now().Add(createExpirationDuration - 5*time.Minute),
+			},
+		},
+		"neverBooted": {
+			Lease: macservice.Lease{
+				LeaseID:             "neverBooted",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 2*time.Hour),
+			},
+		},
+	}
+
+	var mc recordMacServiceClient
+	handleDeadBots(&mc, bots, leases)
+
+	got := mc.vacate
+	want := []macservice.VacateRequest{{LeaseID: "deadLeasePresent"}}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Vacated leases mismatch (-want +got):\n%s", diff)
+	}
+
+	if _, ok := leases["deadLeasePresent"]; ok {
+		t.Errorf("deadLeasePresent present in leases, want deleted")
+	}
+}
+
+func TestRenewLeases(t *testing.T) {
+	// Test leases:
+	// * "new" was created <1hr ago.
+	// * "standard" was created >1hr ago.
+	// * "unmanaged" was created >1hr ago, but is not managed by makemac.
+	//
+	// renewLeases should renew "standard" and none of the others.
+	leases := map[string]macservice.Instance{
+		"new": {
+			Lease: macservice.Lease{
+				LeaseID:             "new",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(createExpirationDuration - 5*time.Minute),
+			},
+		},
+		"standard": {
+			Lease: macservice.Lease{
+				LeaseID:             "standard",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+				Expires:             time.Now().Add(renewExpirationDuration - 5*time.Minute),
+			},
+		},
+		"unmanaged": {
+			Lease: macservice.Lease{
+				LeaseID:             "unmanaged",
+				VMResourceNamespace: macservice.Namespace{ProjectName: "other"},
+				Expires:             time.Now().Add(renewExpirationDuration - 5*time.Minute),
+			},
+		},
+	}
+
+	var mc recordMacServiceClient
+	renewLeases(&mc, leases)
+
+	got := mc.renew
+	want := []macservice.RenewRequest{{LeaseID: "standard", Duration: renewExpirationDurationString}}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Renewed leases mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func TestHandleObsoleteLeases(t *testing.T) {
+	// Test leases:
+	// * "active" uses image "active-image"
+	// * "obsolete" uses image "obsolete-image"
+	// * "unmanaged" uses image "obsolete-image", but is not managed by makemac.
+	//
+	// handleObsoleteLeases should vacate "obsolute" and none of the others.
+	config := []imageConfig{
+		{
+			Name:     "active",
+			Image:    "active-image",
+			MinCount: 1,
+		},
+	}
+	leases := map[string]macservice.Instance{
+		"active": {
+			Lease: macservice.Lease{
+				LeaseID:             "active",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+			},
+			InstanceSpecification: macservice.InstanceSpecification{
+				DiskSelection: macservice.DiskSelection{
+					ImageHashes: macservice.ImageHashes{
+						BootSHA256: "active-image",
+					},
+				},
+			},
+		},
+		"obsolete": {
+			Lease: macservice.Lease{
+				LeaseID:             "obsolete",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+			},
+			InstanceSpecification: macservice.InstanceSpecification{
+				DiskSelection: macservice.DiskSelection{
+					ImageHashes: macservice.ImageHashes{
+						BootSHA256: "obsolete-image",
+					},
+				},
+			},
+		},
+		"unmanaged": {
+			Lease: macservice.Lease{
+				LeaseID:             "obsolete",
+				VMResourceNamespace: macservice.Namespace{ProjectName: "other"},
+			},
+			InstanceSpecification: macservice.InstanceSpecification{
+				DiskSelection: macservice.DiskSelection{
+					ImageHashes: macservice.ImageHashes{
+						BootSHA256: "obsolete-image",
+					},
+				},
+			},
+		},
+	}
+
+	var mc recordMacServiceClient
+	handleObsoleteLeases(&mc, config, leases)
+
+	got := mc.vacate
+	want := []macservice.VacateRequest{{LeaseID: "obsolete"}}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Vacated leases mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func TestAddNewLeases(t *testing.T) {
+	// Test leases:
+	// * "image-a-1" uses image "image-a"
+	// * "unmanaged" uses image "image-a", but is not managed by makemac.
+	//
+	// Test images:
+	// * "image-a" wants 2 instances.
+	// * "image-b" wants 2 instances.
+	//
+	// addNewLeases should create 1 "image-a" instance (ignoring
+	// "unmanaged") and 2 "image-b" instances.
+	config := []imageConfig{
+		{
+			Name:     "a",
+			Image:    "image-a",
+			MinCount: 2,
+		},
+		{
+			Name:     "b",
+			Image:    "image-b",
+			MinCount: 2,
+		},
+	}
+	leases := map[string]macservice.Instance{
+		"image-a-1": {
+			Lease: macservice.Lease{
+				LeaseID:             "image-a-1",
+				VMResourceNamespace: macservice.Namespace{ProjectName: managedProject},
+			},
+			InstanceSpecification: macservice.InstanceSpecification{
+				DiskSelection: macservice.DiskSelection{
+					ImageHashes: macservice.ImageHashes{
+						BootSHA256: "image-a",
+					},
+				},
+			},
+		},
+		"unmanaged": {
+			Lease: macservice.Lease{
+				LeaseID:             "unmanaged",
+				VMResourceNamespace: macservice.Namespace{ProjectName: "other"},
+			},
+			InstanceSpecification: macservice.InstanceSpecification{
+				DiskSelection: macservice.DiskSelection{
+					ImageHashes: macservice.ImageHashes{
+						BootSHA256: "image-a",
+					},
+				},
+			},
+		},
+	}
+
+	var mc recordMacServiceClient
+	addNewLeases(&mc, config, leases)
+
+	got := mc.lease
+	want := []macservice.LeaseRequest{makeLeaseRequest("image-a"), makeLeaseRequest("image-b"), makeLeaseRequest("image-b")}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("Lease request mismatch (-want +got):\n%s", diff)
+	}
+}

go.mod

@@ -29,11 +29,11 @@ require (
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang-migrate/migrate/v4 v4.15.0-beta.3
 	github.com/golang/protobuf v1.5.3
-	github.com/google/go-cmp v0.5.9
+	github.com/google/go-cmp v0.6.0
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-github/v48 v48.1.0
 	github.com/google/safehtml v0.0.3-0.20220430015336-00016cfeca15
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.3.1
 	github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7
@@ -50,7 +50,7 @@ require (
 	github.com/shurcooL/githubv4 v0.0.0-20220520033151-0b4e3294ff00
 	github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07
 	github.com/yuin/goldmark v1.6.0
-	go.chromium.org/luci v0.0.0-20231024170510-08aad78315cf
+	go.chromium.org/luci v0.0.0-20240207061751-3ff7b3e74e1c
 	go.opencensus.io v0.24.0
 	go4.org v0.0.0-20180809161055-417644f6feb5
 	golang.org/x/crypto v0.18.0
@@ -67,8 +67,8 @@ require (
 	golang.org/x/tools v0.17.0
 	google.golang.org/api v0.136.0
 	google.golang.org/appengine v1.6.8-0.20221117013220-504804fb50de
-	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5
+	google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d
-	google.golang.org/grpc v1.58.2
+	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/inf.v0 v0.9.1
 	rsc.io/markdown v0.0.0-20240117044121-669d2fdf1650
@@ -93,18 +93,18 @@ require (
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
-	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fogleman/gg v1.3.0 // indirect
 	github.com/go-fonts/liberation v0.2.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-pdf/fpdf v0.5.0 // indirect
 	github.com/goccy/go-json v0.9.11 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
-	github.com/golang/glog v1.1.0 // indirect
+	github.com/golang/glog v1.1.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -128,6 +128,7 @@ require (
 	github.com/klauspost/asmfmt v1.3.2 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
@@ -146,18 +147,18 @@ require (
 	github.com/shurcooL/graphql v0.0.0-20220520033453-bdb1221e171e // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
-	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
-	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
-	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gonum.org/v1/plot v0.10.0 // indirect
-	google.golang.org/genproto v0.0.0-20230807174057-1744710a1577 // indirect
+	google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
 	google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -212,6 +212,7 @@ github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.20 h1:VIPb/a2s17qNeQgDnkfZC35RScx+blkKF8GV68n80J4=
 github.com/creack/pty v1.1.20/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
@@ -250,8 +251,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF
 github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
 github.com/esimov/stackblur-go v1.1.0 h1:fwnZJC/7sHFzu4CDMgdJ1QxMN/q3k5MGILuoU4hH6oQ=
 github.com/esimov/stackblur-go v1.1.0/go.mod h1:7PcTPCHHKStxbZvBkUlQJjRclqjnXtQ0NoORZt1AlHE=
-github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
-github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
@@ -290,8 +291,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
-github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -343,8 +344,8 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
+github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
-github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
+github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -410,8 +411,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v35 v35.2.0/go.mod h1:s0515YVTI+IMrDoy9Y4pHt9ShGpzHvHO8rZ7L7acgvs=
@@ -452,8 +453,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
 github.com/googleapis/gax-go v0.0.0-20161107002406-da06d194a00e/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@@ -614,8 +615,9 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
 github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
@@ -823,8 +825,8 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
-go.chromium.org/luci v0.0.0-20231024170510-08aad78315cf h1:oTWSnKZi53LF2m0GlyheWtQUJAzs4sqZz6dmGz2Ef54=
+go.chromium.org/luci v0.0.0-20240207061751-3ff7b3e74e1c h1:0khFVd4Exa1DPEwCKjJW7vqI6BS128dTf5zghR3MOqE=
-go.chromium.org/luci v0.0.0-20231024170510-08aad78315cf/go.mod h1:U/+t0vTWBqHr2eOLKGi2nvD0ksAaXJmfjAvy88wmkXo=
+go.chromium.org/luci v0.0.0-20240207061751-3ff7b3e74e1c/go.mod h1:Pxji2l9vIPcilS+otwL6AZLNbNxGTzhuXSf1h53SX64=
 go.mongodb.org/mongo-driver v1.7.0/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -835,18 +837,18 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
-go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 h1:2ea0IkZBsWH+HA2GkD+7+hRw2u97jzdFyRtXuO14a1s=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A=
-go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0/go.mod h1:4m3RnBBb+7dB9d21y510oO1pdB1V4J6smNf14WXcBFQ=
+go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
-go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
-go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
-go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
-go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
-go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
-go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@@ -1325,14 +1327,14 @@ google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm
 google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
 google.golang.org/genproto v0.0.0-20210721163202-f1cecdd8b78a/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
 google.golang.org/genproto v0.0.0-20210726143408-b02e89920bf0/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20230807174057-1744710a1577 h1:Tyk/35yqszRCvaragTn5NnkY6IiKk/XvHzEWepo71N0=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
-google.golang.org/genproto v0.0.0-20230807174057-1744710a1577/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
-google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
-google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577 h1:ZX0eQu2J+jOO87sq8fQG8J/Nfp7D7BhHpixIE5EYK/k=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 h1:wukfNtZmZUurLN/atp2hiIeTKn7QJWIQdHzqmsOnAOk=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
 google.golang.org/grpc v0.0.0-20170208002647-2a6bf6142e96/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1360,8 +1362,8 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
-google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

internal/macservice/client.go

@@ -60,13 +60,22 @@ func (c *Client) do(method, endpoint string, input, output any) error {
 		return fmt.Errorf("response error %s: %s", resp.Status, body)
 	}
 
-	if json.Unmarshal(body, output); err != nil {
+	if err := json.Unmarshal(body, output); err != nil {
 		return fmt.Errorf("error decoding response: %w; body: %s", err, body)
 	}
 
 	return nil
 }
 
+// Lease creates a new lease.
+func (c *Client) Lease(req LeaseRequest) (LeaseResponse, error) {
+	var resp LeaseResponse
+	if err := c.do("POST", "leases:create", req, &resp); err != nil {
+		return LeaseResponse{}, fmt.Errorf("error sending request: %w", err)
+	}
+	return resp, nil
+}
+
 // Renew updates the expiration time of a lease. Note that
 // RenewRequest.Duration is the lease duration from now, not from the current
 // lease expiration time.
@@ -78,6 +87,15 @@ func (c *Client) Renew(req RenewRequest) (RenewResponse, error) {
 	return resp, nil
 }
 
+// Vacate vacates a lease.
+func (c *Client) Vacate(req VacateRequest) error {
+	var resp struct{} // no response body
+	if err := c.do("POST", "leases:vacate", req, &resp); err != nil {
+		return fmt.Errorf("error sending request: %w", err)
+	}
+	return nil
+}
+
 // Find searches for leases.
 func (c *Client) Find(req FindRequest) (FindResponse, error) {
 	var resp FindResponse

internal/macservice/leases.go

@@ -11,6 +11,25 @@ import (
 // These are minimal definitions. Many fields have been omitted since we don't
 // need them yet.
 
+type LeaseRequest struct {
+	VMResourceNamespace Namespace `json:"vmResourceNamespace"`
+
+	InstanceSpecification InstanceSpecification `json:"instanceSpecification"`
+
+	// Duration is ultimately a Duration protobuf message.
+	//
+	// https://pkg.go.dev/google.golang.org/protobuf@v1.31.0/types/known/durationpb#hdr-JSON_Mapping:
+	// "In JSON format, the Duration type is encoded as a string rather
+	// than an object, where the string ends in the suffix "s" (indicating
+	// seconds) and is preceded by the number of seconds, with nanoseconds
+	// expressed as fractional seconds."
+	Duration string `json:"duration"`
+}
+
+type LeaseResponse struct {
+	PendingLease Lease `json:"pendingLease"`
+}
+
 type RenewRequest struct {
 	LeaseID string `json:"leaseId"`
 
@@ -28,6 +47,10 @@ type RenewResponse struct {
 	Expires time.Time `json:"expires"`
 }
 
+type VacateRequest struct {
+	LeaseID string `json:"leaseId"`
+}
+
 type FindRequest struct {
 	VMResourceNamespace Namespace `json:"vmResourceNamespace"`
 }
@@ -44,10 +67,48 @@ type Namespace struct {
 
 type Instance struct {
 	Lease Lease `json:"lease"`
+
+	InstanceSpecification InstanceSpecification `json:"instanceSpecification"`
 }
 
 type Lease struct {
 	LeaseID string `json:"leaseId"`
 
+	VMResourceNamespace Namespace `json:"vmResourceNamespace"`
+
 	Expires time.Time `json:"expires"`
 }
+
+type InstanceSpecification struct {
+	Profile     MachineProfile     `json:"profile"`
+	AccessLevel NetworkAccessLevel `json:"accessLevel"`
+	OSType      OSType             `json:"osType"`
+
+	DiskSelection DiskSelection `json:"diskSelection"`
+}
+
+type DiskSelection struct {
+	ImageHashes ImageHashes `json:"imageHashes"`
+}
+
+type ImageHashes struct {
+	BootSHA256 string `json:"bootSha256"`
+}
+
+type MachineProfile string
+
+const (
+	V1_MEDIUM_VM MachineProfile = "V1_MEDIUM_VM"
+)
+
+type NetworkAccessLevel string
+
+const (
+	GOLANG_OSS NetworkAccessLevel = "GOLANG_OSS"
+)
+
+type OSType string
+
+const (
+	MAC OSType = "MAC"
+)