[medium] Support fetch of instance meta-data for agents running in AWS

agent.go

@@ -36,14 +36,18 @@ type Agent struct {
 
 // AgentEnv stores basic information of the endpoint
 type AgentEnv struct {
-	Init      string   `json:"init,omitempty"`
-	Ident     string   `json:"ident,omitempty"`
-	OS        string   `json:"os,omitempty"`
-	Arch      string   `json:"arch,omitempty"`
-	IsProxied bool     `json:"isproxied"`
-	Proxy     string   `json:"proxy,omitempty"`
-	Addresses []string `json:"addresses,omitempty"`
-	PublicIP  string   `json:"publicip,omitempty"`
+	Init            string   `json:"init,omitempty"`
+	Ident           string   `json:"ident,omitempty"`
+	OS              string   `json:"os,omitempty"`
+	Arch            string   `json:"arch,omitempty"`
+	IsProxied       bool     `json:"isproxied"`
+	Proxy           string   `json:"proxy,omitempty"`
+	Addresses       []string `json:"addresses,omitempty"`
+	PublicIP        string   `json:"publicip,omitempty"`
+	AWSInstanceID   string   `json:"aws_instanceid,omitempty"`
+	AWSLocalIPV4    string   `json:"aws_localipv4,omitempty"`
+	AWSAMIID        string   `json:"aws_amiid,omitempty"`
+	AWSInstanceType string   `json:"aws_instancetype,omitempty"`
 }
 
 type AgentsStats struct {

conf/mig-agent-conf.go.inc

@@ -27,6 +27,9 @@ var MUSTINSTALLSERVICE bool = true
 // attempt to discover the public IP of the endpoint by querying the api
 var DISCOVERPUBLICIP = true
 
+// attempt to discover meta-data for instances running in AWS
+var DISCOVERAWSMETA = true
+
 // in check-in mode, the agent connects to the relay, runs all pending commands
 // and exits. this mode is used to run the agent as a cron job, not a daemon.
 var CHECKIN = false

mig-agent/aws.go

@@ -0,0 +1,130 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Contributor:
+// - Aaron Meihm ameihm@mozilla.com [:alm]
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"mig.ninja/mig"
+	"net"
+	"net/http"
+	"time"
+)
+
+// Various agent functions that are specific to an agent that is running
+// in Amazon Web Services
+
+const AWSMETAIP string = "169.254.169.254"
+const AWSMETAPORT int = 80
+
+// The maximum number of bytes we will fetch in a response from the metadata
+// service
+const FETCHBODYMAX int64 = 10240
+
+func addAWSMetadata(orig_ctx Context) (ctx Context, err error) {
+	ctx = orig_ctx
+
+	// First, check and see if we have access to a valid metadata service
+	buf, err := awsFetchMeta("")
+	if err != nil || buf == "" {
+		ctx.Channels.Log <- mig.Log{Desc: "AWS metadata service not found, skipping fetch"}.Debug()
+		return ctx, nil
+	}
+
+	// Attempt to fetch metadata; if any error occurs we just revert to the
+	// previous context
+	ctx.Channels.Log <- mig.Log{Desc: "Attempting to retrieve AWS instance metadata"}.Debug()
+	flist := []func(Context) (Context, error){
+		addAWSInstanceID,
+		addAWSLocalIPV4,
+		addAWSAMIID,
+		addAWSInstanceType,
+	}
+	for i := range flist {
+		ctx, err = flist[i](ctx)
+		if err != nil {
+			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Error during metadata fetch: %v", err)}.Debug()
+			return orig_ctx, nil
+		}
+	}
+	ctx.Channels.Log <- mig.Log{Desc: "AWS metadata fetch successful"}.Debug()
+	return ctx, nil
+}
+
+func addAWSInstanceID(orig_ctx Context) (ctx Context, err error) {
+	ctx = orig_ctx
+	var res string
+	res, err = awsFetchMeta("instance-id")
+	if err != nil {
+		return
+	}
+	ctx.Agent.Env.AWSInstanceID = res
+	return
+}
+
+func addAWSLocalIPV4(orig_ctx Context) (ctx Context, err error) {
+	ctx = orig_ctx
+	var res string
+	res, err = awsFetchMeta("local-ipv4")
+	if err != nil {
+		return
+	}
+	ctx.Agent.Env.AWSLocalIPV4 = res
+	return
+}
+
+func addAWSAMIID(orig_ctx Context) (ctx Context, err error) {
+	ctx = orig_ctx
+	var res string
+	res, err = awsFetchMeta("ami-id")
+	if err != nil {
+		return
+	}
+	ctx.Agent.Env.AWSAMIID = res
+	return
+}
+
+func addAWSInstanceType(orig_ctx Context) (ctx Context, err error) {
+	ctx = orig_ctx
+	var res string
+	res, err = awsFetchMeta("instance-type")
+	if err != nil {
+		return
+	}
+	ctx.Agent.Env.AWSInstanceType = res
+	return
+}
+
+func awsFetchMeta(endpoint string) (result string, err error) {
+	tr := &http.Transport{
+		Dial: (&net.Dialer{Timeout: time.Second}).Dial,
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(awsMetaURL() + endpoint)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("invalid HTTP response code returned by metadata service")
+		return
+	}
+	if resp.ContentLength == -1 || resp.ContentLength > FETCHBODYMAX {
+		err = fmt.Errorf("invalid content length in response body")
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	result = string(body)
+	return
+}
+
+func awsMetaURL() string {
+	return fmt.Sprintf("http://%v:%v/latest/meta-data/", AWSMETAIP, AWSMETAPORT)
+}

mig-agent/config.go

@@ -21,6 +21,7 @@ type config struct {
 		IsImmortal       bool
 		InstallService   bool
 		DiscoverPublicIP bool
+		DiscoverAWSMeta  bool
 		CheckIn          bool
 		Relay            string
 		Socket           string
@@ -70,6 +71,7 @@ func configLoad(path string) (err error) {
 	ISIMMORTAL = config.Agent.IsImmortal
 	MUSTINSTALLSERVICE = config.Agent.InstallService
 	DISCOVERPUBLICIP = config.Agent.DiscoverPublicIP
+	DISCOVERAWSMETA = config.Agent.DiscoverAWSMeta
 	CHECKIN = config.Agent.CheckIn
 	LOGGINGCONF = config.Logging
 	AMQPBROKER = config.Agent.Relay

mig-agent/context.go

@@ -139,6 +139,14 @@ func Init(foreground, upgrade bool) (ctx Context, err error) {
 		}
 	}
 
+	// Discover AWS instance metadata
+	if DISCOVERAWSMETA {
+		ctx, err = addAWSMetadata(ctx)
+		if err != nil {
+			panic(err)
+		}
+	}
+
 	// find the run directory
 	ctx.Agent.RunDir = getRunDir()
 

tools/standalone_install.sh

@@ -300,6 +300,7 @@ var TAGS = struct {
 var ISIMMORTAL bool = false
 var MUSTINSTALLSERVICE bool = true
 var DISCOVERPUBLICIP bool = false
+var DISCOVERAWSMETA bool = true
 var CHECKIN bool = false
 var APIURL string = "http://localhost:1664/api/v1/"
 var LOGGINGCONF = mig.Logging{