Merge pull request #27565 from rothrock/env-by-regex

[splunk] Use a regex to match environment variables

daemon/logger/fluentd/fluentd.go

@@ -78,7 +78,10 @@ func New(info logger.Info) (logger.Logger, error) {
 		return nil, err
 	}
 
-	extra := info.ExtraAttributes(nil)
+	extra, err := info.ExtraAttributes(nil)
+	if err != nil {
+		return nil, err
+	}
 
 	bufferLimit := defaultBufferLimit
 	if info.Config[bufferLimitKey] != "" {
@@ -172,6 +175,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 	for key := range cfg {
 		switch key {
 		case "env":
+		case "env-regex":
 		case "labels":
 		case "tag":
 		case addressKey:

daemon/logger/gcplogs/gcplogging.go

@@ -17,13 +17,14 @@ import (
 const (
 	name = "gcplogs"
 
-	projectOptKey = "gcp-project"
-	logLabelsKey  = "labels"
-	logEnvKey     = "env"
-	logCmdKey     = "gcp-log-cmd"
-	logZoneKey    = "gcp-meta-zone"
-	logNameKey    = "gcp-meta-name"
-	logIDKey      = "gcp-meta-id"
+	projectOptKey  = "gcp-project"
+	logLabelsKey   = "labels"
+	logEnvKey      = "env"
+	logEnvRegexKey = "env-regex"
+	logCmdKey      = "gcp-log-cmd"
+	logZoneKey     = "gcp-meta-zone"
+	logNameKey     = "gcp-meta-name"
+	logIDKey       = "gcp-meta-id"
 )
 
 var (
@@ -133,6 +134,11 @@ func New(info logger.Info) (logger.Logger, error) {
 		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
 	}
 
+	extraAttributes, err := info.ExtraAttributes(nil)
+	if err != nil {
+		return nil, err
+	}
+
 	l := &gcplogs{
 		logger: lg,
 		container: &containerInfo{
@@ -141,7 +147,7 @@ func New(info logger.Info) (logger.Logger, error) {
 			ImageName: info.ContainerImageName,
 			ImageID:   info.ContainerImageID,
 			Created:   info.ContainerCreated,
-			Metadata:  info.ExtraAttributes(nil),
+			Metadata:  extraAttributes,
 		},
 	}
 
@@ -185,7 +191,7 @@ func New(info logger.Info) (logger.Logger, error) {
 func ValidateLogOpts(cfg map[string]string) error {
 	for k := range cfg {
 		switch k {
-		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey, logZoneKey, logNameKey, logIDKey:
+		case projectOptKey, logLabelsKey, logEnvKey, logEnvRegexKey, logCmdKey, logZoneKey, logNameKey, logIDKey:
 		default:
 			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
 		}

daemon/logger/gelf/gelf.go

@@ -69,12 +69,17 @@ func New(info logger.Info) (logger.Logger, error) {
 		"_created":        info.ContainerCreated,
 	}
 
-	extraAttrs := info.ExtraAttributes(func(key string) string {
+	extraAttrs, err := info.ExtraAttributes(func(key string) string {
 		if key[0] == '_' {
 			return key
 		}
 		return "_" + key
 	})
+
+	if err != nil {
+		return nil, err
+	}
+
 	for k, v := range extraAttrs {
 		extra[k] = v
 	}
@@ -157,6 +162,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case "tag":
 		case "labels":
 		case "env":
+		case "env-regex":
 		case "gelf-compression-level":
 			i, err := strconv.Atoi(val)
 			if err != nil || i < flate.DefaultCompression || i > flate.BestCompression {

daemon/logger/journald/journald.go

@@ -75,7 +75,10 @@ func New(info logger.Info) (logger.Logger, error) {
 		"CONTAINER_NAME":    info.Name(),
 		"CONTAINER_TAG":     tag,
 	}
-	extraAttrs := info.ExtraAttributes(sanitizeKeyMod)
+	extraAttrs, err := info.ExtraAttributes(sanitizeKeyMod)
+	if err != nil {
+		return nil, err
+	}
 	for k, v := range extraAttrs {
 		vars[k] = v
 	}
@@ -89,6 +92,7 @@ func validateLogOpt(cfg map[string]string) error {
 		switch key {
 		case "labels":
 		case "env":
+		case "env-regex":
 		case "tag":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for journald log driver", key)

daemon/logger/jsonfilelog/jsonfilelog.go

@@ -67,7 +67,11 @@ func New(info logger.Info) (logger.Logger, error) {
 	}
 
 	var extra []byte
-	if attrs := info.ExtraAttributes(nil); len(attrs) > 0 {
+	attrs, err := info.ExtraAttributes(nil)
+	if err != nil {
+		return nil, err
+	}
+	if len(attrs) > 0 {
 		var err error
 		extra, err = json.Marshal(attrs)
 		if err != nil {
@@ -122,6 +126,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case "max-size":
 		case "labels":
 		case "env":
+		case "env-regex":
 		default:
 			return fmt.Errorf("unknown log opt '%s' for json-file log driver", key)
 		}

daemon/logger/jsonfilelog/jsonfilelog_test.go

@@ -160,13 +160,13 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
 	}
 	defer os.RemoveAll(tmp)
 	filename := filepath.Join(tmp, "container.log")
-	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl"}
+	config := map[string]string{"labels": "rack,dc", "env": "environ,debug,ssl", "env-regex": "^dc"}
 	l, err := New(logger.Info{
 		ContainerID:     cid,
 		LogPath:         filename,
 		Config:          config,
 		ContainerLabels: map[string]string{"rack": "101", "dc": "lhr"},
-		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true"},
+		ContainerEnv:    []string{"environ=production", "debug=false", "port=10001", "ssl=true", "dc_region=west"},
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -189,11 +189,12 @@ func TestJSONFileLoggerWithLabelsEnv(t *testing.T) {
 		t.Fatal(err)
 	}
 	expected := map[string]string{
-		"rack":    "101",
-		"dc":      "lhr",
-		"environ": "production",
-		"debug":   "false",
-		"ssl":     "true",
+		"rack":      "101",
+		"dc":        "lhr",
+		"environ":   "production",
+		"debug":     "false",
+		"ssl":       "true",
+		"dc_region": "west",
 	}
 	if !reflect.DeepEqual(extra, expected) {
 		t.Fatalf("Wrong log attrs: %q, expected %q", extra, expected)

daemon/logger/logentries/logentries.go

@@ -80,6 +80,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 	for key := range cfg {
 		switch key {
 		case "env":
+		case "env-regex":
 		case "labels":
 		case "tag":
 		case key:

daemon/logger/loginfo.go

@@ -3,6 +3,7 @@ package logger
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 )
@@ -26,7 +27,7 @@ type Info struct {
 // ExtraAttributes returns the user-defined extra attributes (labels,
 // environment variables) in key-value format. This can be used by log drivers
 // that support metadata to add more context to a log.
-func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string {
+func (info *Info) ExtraAttributes(keyMod func(string) string) (map[string]string, error) {
 	extra := make(map[string]string)
 	labels, ok := info.Config["labels"]
 	if ok && len(labels) > 0 {
@@ -40,14 +41,15 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string
 		}
 	}
 
+	envMapping := make(map[string]string)
+	for _, e := range info.ContainerEnv {
+		if kv := strings.SplitN(e, "=", 2); len(kv) == 2 {
+			envMapping[kv[0]] = kv[1]
+		}
+	}
+
 	env, ok := info.Config["env"]
 	if ok && len(env) > 0 {
-		envMapping := make(map[string]string)
-		for _, e := range info.ContainerEnv {
-			if kv := strings.SplitN(e, "=", 2); len(kv) == 2 {
-				envMapping[kv[0]] = kv[1]
-			}
-		}
 		for _, l := range strings.Split(env, ",") {
 			if v, ok := envMapping[l]; ok {
 				if keyMod != nil {
@@ -58,7 +60,23 @@ func (info *Info) ExtraAttributes(keyMod func(string) string) map[string]string
 		}
 	}
 
-	return extra
+	envRegex, ok := info.Config["env-regex"]
+	if ok && len(envRegex) > 0 {
+		re, err := regexp.Compile(envRegex)
+		if err != nil {
+			return nil, err
+		}
+		for k, v := range envMapping {
+			if re.MatchString(k) {
+				if keyMod != nil {
+					k = keyMod(k)
+				}
+				extra[k] = v
+			}
+		}
+	}
+
+	return extra, nil
 }
 
 // Hostname returns the hostname from the underlying OS.

daemon/logger/splunk/splunk.go

@@ -39,6 +39,7 @@ const (
 	splunkGzipCompressionKey      = "splunk-gzip"
 	splunkGzipCompressionLevelKey = "splunk-gzip-level"
 	envKey                        = "env"
+	envRegexKey                   = "env-regex"
 	labelsKey                     = "labels"
 	tagKey                        = "tag"
 )
@@ -235,7 +236,10 @@ func New(info logger.Info) (logger.Logger, error) {
 		}
 	}
 
-	attrs := info.ExtraAttributes(nil)
+	attrs, err := info.ExtraAttributes(nil)
+	if err != nil {
+		return nil, err
+	}
 
 	var (
 		postMessagesFrequency = getAdvancedOptionDuration(envVarPostMessagesFrequency, defaultPostMessagesFrequency)
@@ -538,6 +542,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 		case splunkGzipCompressionKey:
 		case splunkGzipCompressionLevelKey:
 		case envKey:
+		case envRegexKey:
 		case labelsKey:
 		case tagKey:
 		default:

daemon/logger/splunk/splunk_test.go

@@ -25,9 +25,10 @@ func TestValidateLogOpt(t *testing.T) {
 		splunkVerifyConnectionKey:     "true",
 		splunkGzipCompressionKey:      "true",
 		splunkGzipCompressionLevelKey: "1",
-		envKey:    "a",
-		labelsKey: "b",
-		tagKey:    "c",
+		envKey:      "a",
+		envRegexKey: "^foo",
+		labelsKey:   "b",
+		tagKey:      "c",
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -215,8 +216,9 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
 			splunkIndexKey:           "myindex",
 			splunkFormatKey:          splunkFormatInline,
 			splunkGzipCompressionKey: "true",
-			tagKey:    "{{.ImageName}}/{{.Name}}",
-			labelsKey: "a",
+			tagKey:      "{{.ImageName}}/{{.Name}}",
+			labelsKey:   "a",
+			envRegexKey: "^foo",
 		},
 		ContainerID:        "containeriid",
 		ContainerName:      "/container_name",
@@ -225,6 +227,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
 		ContainerLabels: map[string]string{
 			"a": "b",
 		},
+		ContainerEnv: []string{"foo_finder=bar"},
 	}
 
 	hostname, err := info.Hostname()
@@ -295,6 +298,7 @@ func TestInlineFormatWithNonDefaultOptions(t *testing.T) {
 			event["source"] != "stdout" ||
 			event["tag"] != "container_image_name/container_name" ||
 			event["attrs"].(map[string]interface{})["a"] != "b" ||
+			event["attrs"].(map[string]interface{})["foo_finder"] != "bar" ||
 			len(event) != 4 {
 			t.Fatalf("Unexpected event in message %v", event)
 		}

daemon/logger/syslog/syslog.go

@@ -186,6 +186,7 @@ func ValidateLogOpt(cfg map[string]string) error {
 	for key := range cfg {
 		switch key {
 		case "env":
+		case "env-regex":
 		case "labels":
 		case "syslog-address":
 		case "syslog-facility":