feat(config): merge key_groups

closes #1123

Signed-off-by: Jonas Badstübner <jonas.badstuebner@hetzner-cloud.de>

config/config.go

@@ -94,6 +94,7 @@ type configFile struct {
 }
 
 type keyGroup struct {
+	Merge   []keyGroup
 	KMS     []kmsKey
 	GCPKMS  []gcpKmsKey  `yaml:"gcp_kms"`
 	AzureKV []azureKVKey `yaml:"azure_keyvault"`
@@ -184,11 +185,34 @@ type Config struct {
 	OmitExtensions          bool
 }
 
-func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) {
-	var groups []sops.KeyGroup
-	if len(cRule.KeyGroups) > 0 {
-		for _, group := range cRule.KeyGroups {
+func deduplicateKeygroup(group sops.KeyGroup) sops.KeyGroup {
+	var deduplicatedKeygroup sops.KeyGroup
+
+	unique := make(map[string]bool)
+	for _, v := range group {
+		key := fmt.Sprintf("%T/%v", v, v.ToString())
+		if _, ok := unique[key]; ok {
+			// key already contained, therefore not unique
+			continue
+		}
+
+		deduplicatedKeygroup = append(deduplicatedKeygroup, v)
+		unique[key] = true
+	}
+
+	return deduplicatedKeygroup
+}
+
+func extractMasterKeys(group keyGroup) (sops.KeyGroup, error) {
 	var keyGroup sops.KeyGroup
+	for _, k := range group.Merge {
+		subKeyGroup, err := extractMasterKeys(k)
+		if err != nil {
+			return nil, err
+		}
+		keyGroup = append(keyGroup, subKeyGroup...)
+	}
+
 	for _, k := range group.Age {
 		keys, err := age.MasterKeysFromRecipients(k)
 		if err != nil {
@@ -217,6 +241,17 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[
 			return nil, err
 		}
 	}
+	return deduplicateKeygroup(keyGroup), nil
+}
+
+func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) {
+	var groups []sops.KeyGroup
+	if len(cRule.KeyGroups) > 0 {
+		for _, group := range cRule.KeyGroups {
+			keyGroup, err := extractMasterKeys(group)
+			if err != nil {
+				return nil, err
+			}
 			groups = append(groups, keyGroup)
 		}
 	} else {

config/config_test.go

@@ -120,6 +120,125 @@ creation_rules:
       - 'https://baz.vault:8200/v1/baz/keys/baz-key'
 `)
 
+var sampleConfigWithMergeType = []byte(`
+creation_rules:
+  - path_regex: ""
+    key_groups:
+    # key00
+    - hc_vault:
+      - 'https://foo.vault:8200/v1/foo/keys/foo-key'
+    - merge:
+      - merge:
+        - kms:
+          # key01
+          - arn: foo
+            aws_profile: foo
+          pgp:
+          # key02
+          - foo
+          gcp_kms:
+          # key03
+          - resource_id: foo
+          azure_keyvault:
+          # key04
+          - vaultUrl: https://foo.vault.azure.net
+            key: foo-key
+            version: fooversion
+          hc_vault:
+          # key05
+          - 'https://bar.vault:8200/v1/bar/keys/bar-key'
+        - kms:
+          # key06
+          - arn: bar
+            aws_profile: bar
+          pgp:
+          # key07
+          - bar
+          gcp_kms:
+          # key08
+          - resource_id: bar
+          # key09
+          - resource_id: baz
+          azure_keyvault:
+          # key10
+          - vaultUrl: https://bar.vault.azure.net
+            key: bar-key
+            version: barversion
+          hc_vault:
+          # key01 - duplicate#1
+          - 'https://baz.vault:8200/v1/baz/keys/baz-key'
+        kms:
+        # key11
+        - arn: baz
+          aws_profile: baz
+        pgp:
+        # key12
+        - baz
+        gcp_kms:
+        # key03 - duplicate#2
+        # --> should be removed when loading config
+        - resource_id: bar
+        azure_keyvault:
+        # key04 - duplicate#3
+        - vaultUrl: https://foo.vault.azure.net
+          key: foo-key
+          version: fooversion
+        hc_vault:
+        # key13 - duplicate#4 - but from different key_group
+        # --> should stay
+        - 'https://foo.vault:8200/v1/foo/keys/foo-key'
+      - kms:
+        # key14
+        - arn: qux
+          aws_profile: qux
+        # key14 - duplicate#5
+        - arn: baz
+          aws_profile: bar
+        pgp:
+        # key15
+        - qux
+        gcp_kms:
+        # key16
+        - resource_id: qux
+        # key17
+        - resource_id: fnord
+        azure_keyvault:
+        # key18
+        - vaultUrl: https://baz.vault.azure.net
+          key: baz-key
+          version: bazversion
+        hc_vault:
+        # key19
+        - 'https://qux.vault:8200/v1/qux/keys/qux-key'
+      # everything below this should be loaded,
+      # since it is not in a merge block
+      kms:
+      # duplicated key06
+      - arn: bar
+        aws_profile: bar
+      # key20
+      - arn: fnord
+        aws_profile: fnord
+      pgp:
+      # duplicated key07
+      - bar
+      gcp_kms:
+      # duplicated key08
+      - resource_id: bar
+      # key21
+      - resource_id: fnord
+      azure_keyvault:
+      # duplicated key10
+      - vaultUrl: https://bar.vault.azure.net
+        key: bar-key
+        version: barversion
+      hc_vault:
+      # duplicated 'key01 - duplicate#2'
+      - 'https://baz.vault:8200/v1/baz/keys/baz-key'
+      # key22
+      - 'https://fnord.vault:8200/v1/fnord/keys/fnord-key'
+`)
+
 var sampleConfigWithSuffixParameters = []byte(`
 creation_rules:
   - path_regex: foobar*
@@ -340,6 +459,14 @@ func TestLoadConfigFileWithGroups(t *testing.T) {
 	assert.Equal(t, expected, conf)
 }
 
+func TestLoadConfigFileWithMerge(t *testing.T) {
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithMergeType, t), "/conf/path", "whatever", nil)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(conf.KeyGroups))
+	assert.Equal(t, 1, len(conf.KeyGroups[0]))
+	assert.Equal(t, 22, len(conf.KeyGroups[1]))
+}
+
 func TestLoadConfigFileWithNoMatchingRules(t *testing.T) {
 	_, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithNoMatchingRules, t), "/conf/path", "foobar2000", nil)
 	assert.NotNil(t, err)