[medium] change agent proxy behavior to prefer proxies

If any proxies have been configured, try those first before attempting
direct connections. This also adds specifying proxies in the agent
config file, where as before they were only compile time options in the
built-in configuration.

Resolves #249

conf/mig-agent-conf.go.inc

@@ -59,7 +59,10 @@ var APIURL string = "http://localhost:1664/api/v1/"
 
 // if the connection still fails after looking for a HTTP_PROXY, try to use the
 // proxies listed below
-var PROXIES = [...]string{`proxy.example.net:3128`, `proxy2.example.net:8080`}
+var PROXIES = []string{"proxy.example.net:3128", "proxy2.example.net:8080"}
+// If you don't want proxies in the built-in configuration, use the following
+// instead.
+// var PROXIES = []string{}
 
 // local socket used to retrieve stat information from a running agent
 var SOCKET = "127.0.0.1:51664"

conf/mig-agent.cfg.inc

@@ -20,6 +20,11 @@
     ; will attempt to restart itself instead of just shutting down
     isimmortal       = on
 
+    ; comma delimited list of host:port proxies to use, if desired
+    ; the agent will attempt to try to proxies for public ip retrieval
+    ; and the relay connection before a direct connection
+    ; proxies          = "proxy1:8888,proxy2:8888"
+
     ; installservice orders the agent to deploy a service init configuration
     ; and start itself during the endpoint's boot process
     installservice   = on

doc/configuration.rst

@@ -856,11 +856,13 @@ public IP during startup.
 Proxy support
 ~~~~~~~~~~~~~
 
-The agent supports connecting to the relay via a CONNECT proxy. It will attempt
+The agent supports connecting to the relay via a CONNECT proxy. If proxies are
-a direct connection first, and if this fails, will look for the environment
+configured, it will attempt to use them before attemping a direct connection. The
-variable `HTTP_PROXY` to use as a proxy. A list of proxies can be manually
+agent will also attempt to use any proxy noted in the environment via the
-added to the configuration of the agent in the `PROXIES` parameters. These
+`HTTP_PROXY` environment variable. A list of proxies can be manually
-proxies will be used if the two previous connections fail.
+added to the configuration of the agent in the `PROXIES` parameters. Proxies can
+also be specified in the agent configuration file, and will override any built-in
+configuration.
 
 An agent using a proxy will reference the name of the proxy in the environment
 fields of the heartbeat sent to the scheduler.
@@ -1635,7 +1637,6 @@ The following parameters are **not** controlable by the configuration file:
 
 * list of investigators public keys in `PUBLICPGPKEYS`
 * list of access control lists in `AGENTACL`
-* list of proxies in `PROXIES`
 
 All other parameters can be overriden in the configuration file. Check out the
 sample file `mig-agent.cfg.inc` in the **conf** folder.

mig-agent/agentcontext/env.go

@@ -44,13 +44,12 @@ func findPublicIP(orig_ctx AgentContext, hints AgentContextHints) (ctx AgentCont
 		Dial: (&net.Dialer{Timeout: 10 * time.Second}).Dial,
 	}
 	client := &http.Client{Transport: tr}
-	resp, err := client.Get(hints.APIUrl + "/ip")
+	var resp *http.Response
-	if err != nil {
+
-		logChan <- mig.Log{Desc: fmt.Sprintf("Public IP retrieval from API failed. Trying with proxy next. Error was: %v", err)}.Info()
+	// If any proxies have been configured, try to use those first, fall back to a
-	} else {
+	// direct connection.
-		goto parseBody
-	}
 	for _, proxy := range hints.Proxies {
+		logChan <- mig.Log{Desc: fmt.Sprintf("Trying proxy %v for public IP retrieval", proxy)}.Debug()
 		pu, err := url.Parse("http://" + proxy)
 		if err != nil {
 			logChan <- mig.Log{Desc: fmt.Sprintf("Failed to parse proxy url http://%s - %v", proxy, err)}.Info()
@@ -65,6 +64,18 @@ func findPublicIP(orig_ctx AgentContext, hints AgentContextHints) (ctx AgentCont
 			goto parseBody
 		}
 	}
+
+	// Try a direct connection, but also take into consideration any proxies that may
+	// have been configured in the proxy related environment variables.
+	logChan <- mig.Log{Desc: "Trying proxy from environment otherwise direct connection for public IP retrieval"}.Debug()
+	tr.Proxy = http.ProxyFromEnvironment
+	resp, err = client.Get(hints.APIUrl + "/ip")
+	if err != nil {
+		logChan <- mig.Log{Desc: fmt.Sprintf("Public IP retrieval from API failed. Error was: %v", err)}.Info()
+	} else {
+		goto parseBody
+	}
+
 	// exit here if no connection succeeded
 	logChan <- mig.Log{Desc: fmt.Sprintf("Failed to retrieve public ip from api: %v", err)}.Err()
 	return

mig-agent/config.go

@@ -9,6 +9,7 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
+	"strings"
 	"time"
 
 	"mig.ninja/mig"
@@ -23,6 +24,7 @@ type config struct {
 		DiscoverPublicIP bool
 		DiscoverAWSMeta  bool
 		CheckIn          bool
+		Proxies          string
 		Relay            string
 		Socket           string
 		HeartbeatFreq    string
@@ -80,6 +82,9 @@ func configLoad(path string) (err error) {
 	MUSTINSTALLSERVICE = config.Agent.InstallService
 	DISCOVERPUBLICIP = config.Agent.DiscoverPublicIP
 	DISCOVERAWSMETA = config.Agent.DiscoverAWSMeta
+	if config.Agent.Proxies != "" {
+		PROXIES = strings.Split(config.Agent.Proxies, ",")
+	}
 	CHECKIN = config.Agent.CheckIn
 	LOGGINGCONF = config.Logging
 	AMQPBROKER = config.Agent.Relay

mig-agent/context.go

@@ -215,29 +215,35 @@ func Init(foreground, upgrade bool) (ctx Context, err error) {
 
 	connected := false
 	// connect to the message broker
-	ctx, err = initMQ(ctx, false, "")
+	//
-	if err != nil {
+	// If any proxies have been configured, we try to use those first. If they fail, or
-		ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay directly: '%v'", err)}.Debug()
+	// no proxies have been setup, just attempt a direct connection.
-		// if the connection failed, look for a proxy
+	for _, proxy := range PROXIES {
-		// in the environment variables, and try again
+		ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Trying proxy %v for relay connection", proxy)}.Debug()
-		ctx, err = initMQ(ctx, true, "")
+		ctx, err = initMQ(ctx, true, proxy)
 		if err != nil {
-			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay using HTTP_PROXY: '%v'", err)}.Debug()
+			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay using proxy %s: '%v'", proxy, err)}.Info()
-			// still failing, try connecting using the proxies in the configuration
+			continue
-			for _, proxy := range PROXIES {
-				ctx, err = initMQ(ctx, true, proxy)
-				if err != nil {
-					ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay using proxy %s: '%v'", proxy, err)}.Debug()
-					continue
-				}
-				connected = true
-				goto mqdone
-			}
-		} else {
-			connected = true
 		}
-	} else {
 		connected = true
+		goto mqdone
+	}
+	// Try and proxy that has been specified in the environment
+	ctx.Channels.Log <- mig.Log{Desc: "Trying proxies from environment for relay connection"}.Debug()
+	ctx, err = initMQ(ctx, true, "")
+	if err == nil {
+		connected = true
+		goto mqdone
+	} else {
+		ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay using HTTP_PROXY: '%v'", err)}.Info()
+	}
+	// Fall back to a direct connection
+	ctx.Channels.Log <- mig.Log{Desc: "Trying direct relay connection"}.Debug()
+	ctx, err = initMQ(ctx, false, "")
+	if err == nil {
+		connected = true
+	} else {
+		ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Failed to connect to relay directly: '%v'", err)}.Info()
 	}
 mqdone:
 	if !connected {

tools/standalone_install.sh

@@ -302,7 +302,7 @@ var LOGGINGCONF = mig.Logging{
     File:   "/var/cache/mig/mig-agent.log",
 }
 var AMQPBROKER string = "amqp://agent:$mqpass@localhost:5672/mig"
-var PROXIES = [...]string{``}
+var PROXIES = []string{}
 var SOCKET string = "127.0.0.1:51664"
 var HEARTBEATFREQ time.Duration = 30 * time.Second
 var REFRESHENV time.Duration = 60 * time.Second