diff --git a/README.rst b/README.rst index d5f849d87..02c15ce1f 100644 --- a/README.rst +++ b/README.rst @@ -282,29 +282,34 @@ file and saves it when done. Upon save, sops browses the entire file as of a key/value tree. Every time sops encounters a leaf value (a value that does not have children), it encrypts the -value with AES256_GCM using the data key, a 256 bits random initialization vector -and 256 bits of random additional data. While the same data key is used to -encrypt all values of a document, each value receives a unique initialization -vector and unique authentication data. +value with AES256_GCM using the data key and a 256 bits random initialization +vector. + +Each file uses a single data key to encrypt all values of a document, but each +value receives a unique initialization vector and has unique authentication data. + +Additional data is used to guarantee the integrity of the encrypted data +and of the tree structure: when encrypting the tree, key names are concatenated +into a byte string that is used as AEAD additional data (aad) when encrypting +the value. The `aad` field is not stored with the value but reconstructed from +the tree structure every time. The result of AES256_GCM encryption is stored in the leaf of the tree using a -simple key/value format:: +base64 encoded string format:: ENC[AES256_GCM, data:CwE4O1s=, iv:S0fozGAOxNma/pWDUuk1iEaYw0wlba0VOLHjPxIok2k=, - aad:nEVizsMMyBXOxySnOHw/trTFBSW72nh+Q80YU7TPgIo=, tag:XaGsYaL9LCkLWJI0uxnTYw==] where: * **data** is the encrypted value * **iv** is the 256 bits initialization vector -* **aad** is the 256 bits additional data * **tag** is the authentication tag The encrypted file is written to disk with nested keys in cleartext and -encrypted values. We expect that keys do not carry sensitive information, and +values encrypted. We expect that keys do not carry sensitive information, and keeping them in cleartext allows for better diff and overall readability. Any valid KMS or PGP master key can later decrypt the data key and access the @@ -316,6 +321,14 @@ is to have two KMS master keys in different region and one PGP public key with the private key stored offline. If, by any chance, both KMS master keys are lost, you can always recover the encrypted data using the PGP private key. +Message Authentication Code +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to authenticating branches of the tree using keys as additional +data, sops computes a MAC on all the values to ensure that no value has been +added or removed fraudulently. The MAC is stored encrypted with AES_GCM and +the data key under tree->`sops`->`mac`. + Threat Model ------------ diff --git a/example.json b/example.json index a4e98c500..f32012b4f 100644 --- a/example.json +++ b/example.json @@ -1,41 +1,43 @@ { + "firstName": "ENC[AES256_GCM,data:Sf9dCw==,iv:OtsxqCFAvsDfiUIu+FmMT+9SZZ+hwFXxWAoA/fFt4n0=,tag:UJpAMAlrLm+TMIAkQCShTg==,type:str]", + "lastName": "ENC[AES256_GCM,data:8CSE1Fc=,iv:ZwNczZao5fK44uYH+TU+RwXSC6OHjbBWCrQiO97Ws3I=,tag:ghRiOGqNPvotGEIE80XpRg==,type:str]", + "age": "ENC[AES256_GCM,data:hXI=,iv:er2l2sivl6Q00VwfF/YTOSs19q3AYqwVCSdBZkUZ9cc=,tag:U9IQcgO8nRdZGKuk/GaB+w==,type:int]", "address": { - "city": "ENC[AES256_GCM,data:2wNRKB+Sjjw=,iv:rmATLCPii2WMzcT80Wp9gOpYQqzx6juRmCf9ioz2ZLM=,aad:dj0QZW0BvZVjF1Dn25hOJpcwcVB0qYvEIhGWgxq6YzQ=,tag:wOoPYU+8BA9DiNFlsal3Aw==,type:str]", - "postalCode": "ENC[AES256_GCM,data:xwWZ/np9Gxv3CQ==,iv:OLwOr7iliPyWWBtKfUUH7E1wQlxJLA6aFxIfNAEC/M0=,aad:8mw5NU8MpyBlrh7XaUqa642jeyJWGqKvduaQ5bWJ5pc=,tag:VFmnc4Ay+yKzyHcrKeEzZQ==,type:str]", - "state": "ENC[AES256_GCM,data:3jY=,iv:Y2bEgkjdn91Pbf5RgJMbyCsyfhV7XWdDhe8wVwTQue0=,aad:DcA5kW1rrET9TxQ4kn9jHSpoMlkcPKs5O5n9wZjZYCQ=,tag:ad1xdNnFwkqx/8EOKVVHIA==,type:str]", - "streetAddress": "ENC[AES256_GCM,data:payzP57DGPl5S9Z7uQ==,iv:UIz34fk9zH4z6hYfu0duXmAnI8CqnoRhoaIUqg1YoYA=,aad:hll9Baw40lMjwj7HePQ1o1Lsuh1LCwrE6+bkG4025sg=,tag:FDBhYxMmJ1Wj/uxYxdvVZg==,type:str]" + "city": "ENC[AES256_GCM,data:DKo/DSI8QjU=,iv:ZVT8sB8Lq7Q1l4kRmEpjq78BLXL6VSG5Wl+s0skKz9k=,tag:VQ8t+CtkTd6l20wrQHECPA==,type:str]", + "postalCode": "ENC[AES256_GCM,data:DxjkWjslhRKFeA==,iv:jZYRetIj1Brxj0Dhc6e06NOwQt4nR0wW6iVRN/n5SwI=,tag:L2fq3kQuNyJ04nGGPpJV9Q==,type:str]", + "state": "ENC[AES256_GCM,data:haM=,iv:dZlMji6974EpdMsW+ZF6kGt4cUG2jJiz1mANZLZaMhU=,tag:XMC1iixS6IRf3zMCf5/ZDw==,type:str]", + "streetAddress": "ENC[AES256_GCM,data:KnPa8Gihd9+dHcXZZg==,iv:KA/JWp/fW0BaTvRlc0SHYZPtdVU6Jzryp8L5CHo1a4I=,tag:t2rp4iR0+VtHvNgBgQ/+OQ==,type:str]" }, - "age": "ENC[AES256_GCM,data:4Y4=,iv:hi1iSH19dHSgG/c7yVbNj4yzueHSmmY46yYqeNCoX5M=,aad:nnyubQyaWeLTcz9k9cMHUlgTwVDMyHf32sWCBm7KWAA=,tag:4lcMjstadzI8K40BoDEfDA==,type:int]", - "firstName": "ENC[AES256_GCM,data:KVe8Dw==,iv:+eg+Rjvaqa2EEp6ufw9c4hwWwObxRLPmxx3fG6rkyps=,aad:3BdHcorHfbvM2Jcs96zX0JY2VQL5dBNgy7zwhqLNqAU=,tag:5OD6MN9SPhBmXuA81hyxhQ==,type:str]", - "lastName": "ENC[AES256_GCM,data:1+koqsI=,iv:b2kBxSW4yOnLFc8qoeylkMtiO/6qr4cZ5VTntXTyXO8=,aad:W7HXukq3lUUMj9i57UehILG2NAp8XCgJMYbvgflWJIY=,tag:HOrgi1L+IRP+X5JGMnm7Ig==,type:str]", "phoneNumbers": [ { - "number": "ENC[AES256_GCM,data:Oo0IxdtBrnfE+bTf,iv:tQ1E/JQ4lHZvj1nQnGL2sKE30sCctjiMCiagS2Yzch8=,aad:P+m5gD3pKfNEOy6t61vbKhEpPtMFI2NZjBPrD/m8T9w=,tag:6iRMUVUEx3UZvUTGTjCdwg==,type:str]", - "type": "ENC[AES256_GCM,data:M3zOKQ==,iv:pD9RO4BPUVu6AWPo2DprRsOqouN+0HJn+RXQAXhfB2s=,aad:KFBBVEEnSjdmah3i2XmPx7wWEiFPrxpnfKYW4BSolhk=,tag:liwNnip/L6SZ9srn0N5G4g==,type:str]" + "number": "ENC[AES256_GCM,data:qgbbyAXoBbkDr1bA,iv:Y8Z2nBp2yV6ldfAU9Zjsb6gCBLQrNMEqvkwSSZ3Y2Z4=,tag:UZksExiLkAALhZ9w5cJ3qw==,type:str]", + "type": "ENC[AES256_GCM,data:29QEQA==,iv:x+GSbhrTvvNj46Kv1FE1bghPBBAm37sLJVMuclg1OnM=,tag:wDS+sfKLDusWlMgpWidRyA==,type:str]" }, { - "number": "ENC[AES256_GCM,data:BI2f/qFUea6UHYQ+,iv:jaVLMju6h7s+AlF7CsPbpUFXO2YtYAqYsCIsyHgfrfI=,aad:N+8sVpdTlY5I+DcvnY018Iyh/QesD7bvwfKHRr7q2L0=,tag:hHPPpQKP4cUIXfh9CFe4dA==,type:str]", - "type": "ENC[AES256_GCM,data:EfKAdEUP,iv:Td+sGaS8XXRqzY98OK08zmdqsO2EqVGK1/yDTursD8U=,aad:h9zi8s+EBsfR3BQG4r+t+uqeChK4Hw6B9nJCrValXnI=,tag:GxSk1LAQIJNGyUy7AvlanQ==,type:str]" + "number": "ENC[AES256_GCM,data:z+CGrbAnrTwABu8b,iv:w5BfJFjJtIoXtTbkhhbRsGNP9cvhYiRIzhxay6WIjbs=,tag:JlwuErBmnbmlkWW6oIgdcQ==,type:str]", + "type": "ENC[AES256_GCM,data:e/dNOAmq,iv:ZFoDttfnZIeHnDfbIzT9t2UgLK/0Bf3oFJ1CmN+Ovco=,tag:oWSUu9exCqZakfaHOeWE3g==,type:str]" } ], "sops": { + "mac": "ENC[AES256_GCM,data:kYZgjW5lpTuTPZOs/rJTKajAxF9yAGRiRZzCFYuRR1UPkLeOzBlh4qLBngw1eu3wLkjJo+Aj6zPuVsZAOHlwGRlYsNzhzMFMTuuNaG2bN8FNl7Z/0qz/LIsuLov3RdaGyC9u++Vsm3NWoAsDLyKATS73F54SrakysIBCK1q4aZk=,iv:+fYGJ0hx/e9eR+i7Un4Ogvf6x5H+6mK8r90f9P9GLdA=,tag:1DV1d7qGh7zP2ZWwi+//uA==,type:str]", + "attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", "kms": [ { - "arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e", - "created_at": 1443204393.48012, - "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwBpvXXfdPzEIyEMxICARCAOy57Odt9ngHHyIjVU8wqMA4QszXdBglNkr+duzKQO316CRoV5r7bO8JwFCb7699qreocJd+RhRH5IIE3" + "created_at": 1444233149.795402, + "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwO9pCAxCN0oznQ7x8CARCAOxrIQYZ7J8/aCCnLUf0zLqL96AwfyYS76+g51sLaQlNTMqNGfslT6cZmw24CdsNrvtz8QypP74+pM7Xd", + "arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e" }, { - "arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d", - "created_at": 1443204394.74377, - "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwag3w44N8+0WBVySwCARCAOzpqMpvzIXV416ycCJd7mn9dBvjqzkUDag/zHlKse57uNN7P0S9GeRVJ6TyJsVNM+GlWx8++F9B+RUE3" + "created_at": 1444233151.305619, + "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxyg2xy9gTYriI3dBgCARCAO2NVWrAab3DY5GdcLzNxTm8wKkyn/8km/5mxGWZX5zerOgZjXsyFAUW9plckQjRAe1JeXbSjhZq5ev/k", + "arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d" } ], "pgp": [ { - "created_at": 1443204394.748745, - "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgAQ//dpZVlRD9WGvz6Pl+PRKvBf661IHLkCeOq5ubzqLIJZu7\nJMNu0KBoO0qX+rgIQtzMU+04QlbIukw01q9ELSDYjBDQPRQJ+6OAeauawxf5mPGa\nZKOaSuoCuPbfOmGj8AENdSCpDaDz+KvOPvo5NNe16kC8BeerFJGewyEwbnkx5dxZ\ngk+LJBOuWRVUEzjsB1pzGfGRzvuzHcrUzWAoA8N936hDFIpoeDYC/8KLc0CWTltA\nYYGaKh5cZxC0R0TgQ5S9GjcU2nZjhcL94XRxZ+9BZDLCDRnjnRfUpPSTHoxr9wmR\nAuLtgyCIolrPl3fqRLJSLUH6FyTo2CO+2mFSx7y9m2OXkKQd1z2tkOlpC9PDTjGT\nVfGvy9nMUsmrgWG35soEmk0nNJaZehiscvZfomBnnHQgqx7DMSMxAnBneFqjsyOQ\nGK7Jacs/tigxe8NZcYhx+usITeQzVLmuqZ2pO5nEGyq0XJhJjxce9YVaeC4QOft0\nlm6qq+m6oABOdKTGh6zuIiWxU1r417IEgV8mkwjlraAvNNPKowQq5j8dohG4HaNK\nOKoOt8aIZWvD3HE9szuH+uDRXBBEAbIvnojQIyrqeIYv1xU8hDTllJPKw/kYD6nx\nMKrw4UAFand5qAgN/6QoIrOPXC2jhA2VegXkt0LXXSoP1ccR4bmlrGRHg0x6Y8zS\nXAE+BVEMYh8l+c86BNhzVOaAKGRor4RKtcZIFCs/Gpa4FxzDp5DfxNn/Ovrhq/Xc\nlmzlWY3ywrRF8JSmni2Asxet31RokiA0TKAQj2Q7SFNlBocR/kvxWs8bUZ+s\n=Z9kg\n-----END PGP MESSAGE-----\n", - "fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21" + "fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21", + "created_at": 1444233151.309663, + "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgARAAgQdMpnTNMCdbdFRpBsC9kxi334LbBrFUkp5lI+YzutZy\nSic85ea06FGL3O93tII9mwGAsESwKlN4nX0d31vuh/lYxMDakyd1IK/BkMG4Z1xG\n52MsACG/pyitMBXkIIyjmR0tVR+CixDsy5cUJxoWq+mfuE2ywziPY+KbEZ50hFXg\naAdKCdInXlLHdId+aXhThhXUGN1seQjtdyZjVXnp8c9hHS2YQdyp/SZf47NJ4A2y\nkO40kNS4oaHUUZIZLtzaFhWytZlpWEJJkIgH/vefL3jLW4SiIiqz24wr7MncsF+A\np8Pteulc5VrvA5CzQIq9qF3Zwn9HV2a0KWLZ/J29EYzSM8u9HLOYqsmNKt0TcVbX\n6eoG3JTJoRDrzO0DZvR3pMm4gQ0WXzHKzpu8g+JYnoQ19AMWJAPbTp5ej3MWHcXD\nXFjz4gsSYbwc4h/zVBOWsYoHlyTLUMwg2BA1YiL89xs8MIhIHOAmvM0mv+QuZQ7S\nCfc1mS04CZSmJvTcNkvE5n76n2iXs6nYNk8TYyQlhYebuQmJQKJuUYjKIHhuxZFa\n30WaSGnKHqIQn1pl7jqyqm8sVTzaKMyhbM0T+UQUJhXcWVr7r+CtRAt8XjVnJMvo\nviJwTWy1Ddo0Vu1licMFJXMnQbQlVh+CZS6FHqcbxfPaYfe7JldGmhwKg+F/NEHS\nXgEf78iLm3FNb4yeOkB/z2xjiZ3XvUAQjsUK5ofF1CJYcQ//YIFex1oO55Z0+qIt\njdDtqivLgf4SFRf0uhOxUrQNuFAvY361F1mvrGPcTubh/Ygq0aVzWzgC9gn7DTo=\n=uQw0\n-----END PGP MESSAGE-----\n" } ] } diff --git a/example.txt b/example.txt index cd46b47a7..85104d50d 100644 --- a/example.txt +++ b/example.txt @@ -1,2 +1,2 @@ -ENC[AES256_GCM,data:jO/n1M+TNC65M7OMHbwo5dgwaHBsoyeUvgIqUjsR38HCqwkHd8uEOhjICetibt46GGoUX0W/x4NS2zU0f0dQg0r+ac5xO7IDJQG6EgMvTMjguae6yQrrGotNv8i6Vg4Ci3SS22G8m96YGsyvDqwI0sPCxW2HkLFWi1zjVpNtZE8cjU6NiEJ51B4HqvTvklVkqMkyN0j15knlh5Dri3Ys2UT4ZkrskkHStr+bF0JI4/iEmr6GhhIWU0CzVgontYhUJWV914HkeHI3rF+dSu2UXmYkijZwHn9XDtXGisvwMMXiwzonF22tNUGa+ZXQUD9XtbMIbh7jyKUCJVjXQVhXxffvctv8maF9fnfHZljNCVPlkrYDzRzA3Fbk70uGV8Yez8TOUvREG/sEJJ+BdEgm8WgEs0TZlH/Yzk5bAJ7f38WKzMBs2WQWBWp65KkWcbfg23m7o/PePbij5P5HK1qpIpb6ay7/Db0Imf3FIZVnS7Xj/N/ECv0qDuX1LFCrdKYaizp1KoXRTiQky0S3IZdMs7C2jtwUhWYdPSp1f/w7BlOMzNtr0hU9NaSzo6SoBgGCMMTsIadK897b5ufFaJ54jMblQ288ngxTey/vfETSjRdKqC//9uNjuaW3NLn8p9PUsxz660rYhJs4w8VHyIThKvfGtwROAbi20szBlxGr7BPeiw+r9rXBl0RWtgRfBoI1Qa6+h9Swo4d/vxXTkhwasZOLfxicO2jdHM6SacsOyD5/Lm31h3SYK/oy6xZrCUnx7X63l0ST8qZnvqfLXua1IS2hRWhrcoC6o/5tIR7P9piOMwsSOkdqjcRGwm1pxTnr+i4uoM/JrHiGHkn++JEShcyPSnXegasucg==,iv:Y4m/FsQ0fUdO836x73hyRlJbLcn+tsTDCaKjcZUm7cg=,aad:zxaIUh+9wW1V9agSfTORFCtJAAzlysQL7xgrERpLPVQ=,tag:HcVAHdDQLIP/snAL8hH+pQ==] -SOPS={"attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", "kms": [{"arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e", "created_at": 1443203537.040281, "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyW6JtWKX5xiAenHt8CARCAO0wv8V2HIJp6ClzQG0TWhzgulinLUoHffRYGVkN7WIFCFkFj8fo5EqCMH0s9/nAt930FhHln82698aCc"}, {"arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d", "created_at": 1443203538.64956, "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAz4RIBXvyNzzptDgTQCARCAO2+Fz1fIkpJ8/8jUf7kEzM6D+Th0V1gswRgZIt7OVLjlID3DuvezmiTFLI3qHRutR8TIwj5K30tXjMs1"}], "pgp": [{"created_at": 1443203538.653892, "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgAQ//Y4rMS49ZSXE/VyU5RYuI2wjvwwcqj5Dn2DB0qxpw0Ec7\nKRgDqqG1Ds03bJ/mNaPhLTOFp3Y132F6Jiy2JqxHATlbYKbFrUUAFUtriX9mMzde\nPdywJN3cC/5BJJmUAZvmKGtVOQSbqTrzgNvfA0SBTfB5v3iuCXA6G9Fm1MwOWy64\nhAlHbZrpuMqkElL0lkDx3beIx97Sr6wjsWP+rSZbfcHBVpg5iBHZi4MyLGJfab+H\n9TFNpy6XG0twC58lhqzZe1fxFQuHzra16TpQyrrl39La0PaE0YK/OKwLo6cdn8Ur\n8v2HuWme7JlUhjW250PMz1c/sbXygPranRUExjDLN4E6ePCuVkdq/A292rBhHOzy\nMzoAI8SiIK+cfCdN2t1KG+E60EX5vHyYgTuJPZ5miZlPFIYCGR7XO0ynuD1Gzh78\nmtFr+6wgdo420tyjQExHm2oq6O/NtjQuf9a05TNAnrQ4cXo89D28TCMtJyizYT0U\nmZwq9zp4Xcu3fg60UrKdpnA1xum5NRpq/kpIjRGnkErAxN7HrrEMjfD3ECLPUsdU\nMHL2DzWmG4DjJVEIXATbDylkYwp/zMzf8lu/9vp7ocwGsyZb/wg9ib9u5/RscFqf\nvfmO8pDWGe6M+1hmGiTspKBP8oOxCt3lkpj1BKzYxGT421DHIpVawk80oabT11TS\nXAHNIbqsCJBrvY3TywR9LwO4PsFPZhsz8Oosg4446XCOC+6jFONw+qCqj/daQQo3\nL+NoLhs5rpgS86MhzY7mdW9TtDr+oFI8gax1SM+gvpUupV1XBWvXAOifgAYX\n=5896\n-----END PGP MESSAGE-----\n", "fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21"}]} \ No newline at end of file +ENC[AES256_GCM,data:qe/cYirOA9x9zFw8AtonYzcRT+Sr9KEMtiRN55YOn1SNBEX7sE9s/M09KSfneKCbHIsNVdh5902vYNAwSFpJ6NhzO/6YR65ucwS1eBx4EQ7s3PYkEWHipgcibW0hXiNvEJGpR/4ZQ/er/xns2cpCUAgpv07Ry9DiGW+us4HM+gA5dplTpYUCdzMl7zGYoPYcIiJVHjkjaHK70w==,iv:v2BgZ6Oa895KpYAeikA0c9g9R8wcu2YOe2kp6vPEMX8=,tag:zQ/C7u2+veEuOVLSAEc1Mg==,type:str] +SOPS={"attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", "kms": [{"arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e", "created_at": 1444233233.692422, "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxatks17s0ZWQIyPi8CARCAO65vxmVs4SOASbNDdnwdeOlg75rz7oeqWId2JyQU8sNyz7+TNvvsLIjIR50AGMwnbMIgTmbM99LDi6Vo"}, {"arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d", "created_at": 1444233235.129884, "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzI3YJKPROE+fG2vJYCARCAO2IeX+3IeMkOOOsQauVrUTP9FVFmcmpXYDT41PDt8nhFvU/Q9RUUoVG1OLeWK+KBDZgu1NWGeUTN3TTs"}], "mac": "ENC[AES256_GCM,data:1Xhgsh64dJ2FM8AhbrRPVik2iXAuZV/kwHD93cConQwJDZw96joPn3S9jD//YcawPiniy4hjOoTxdkTcjdmy0NCGB3VUbetKDRUGgWatTvVMw2msL6UZYkCaqj6c66KgtUHPCgsr0GhsTOCN/F4+hg3qlLykJ6DJVLck0HZ7K9I=,iv:RL0nykJwmnRWXegD1VUQ5fNC8i2AnJHK/Y3jorfwYMI=,tag:JFRq575pT0jjk4xAwkMfog==,type:str]", "pgp": [{"created_at": 1444233235.135862, "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgAQ/9GNa5B4AkO7UODicvjpsgEGLd++1mJteKOwww/08src+H\nnfe/VtTvOdCNVNwvkeKtANvM5DCX9RVTjul4SH7iKd/O9XmTFXA66fhgAbRmEczm\npzQXog/res0u/q+mVwdSDqx/6qBViIcz1Zgc5oFnAneRlAke2/UsNFuFbtaQDZZh\nuralZFdrLx/DWjqEWXEh9D+caek2z/Tjhl/PQ6JNPEa7aZfMLjuTuaoPkSgd87Zc\ndnz/UL77Wx1zdv/cLtO2XvJhOvi0BF9dkg4evouTtNJs+WjQvkBCAijwdC5JdjTz\nWj4mV4H/YdlOn+j2ng3GGmF6GIX5x9FLLD5a9PjSgHVvAH8ZpXkCVY2U8e7QAW7+\nv3KLKGZFWvke62pmypj3777Z5MBj/SJAlzmuPdCLQCXIIpozqK4N4qTvg4Rt5TsN\n8YH9HYfWhX6fHvd67alwrz4IV3g1LgCKCGQd0EXl8pjYwErspGym3UOyZKSD4dDb\nH8zdbr2bQxZ2dJR3o+DVTdohfFjxUqHAZ8bO3vkUT4xblY8n2NnIUWxw3tDHdV/6\niXWVfRcgsIRmFM8qZ7CwwxDZFgLGY3oPhzNmze+B1g5xMG/l4MbKwjCb2EQ38CDr\nDG11GMG5ewhZUDwry4aDpxQMUhvuLBupve+caHzs62zTyWxurwLwfzOHbUyCxbjS\nXAEQ++zCoKncWsAxJdaoIvAvTJBEJeRyGToPESe8iYjmkT1jYZCMj30opOmOZ94M\nE0X4OYpb8FGL/QhOASMe8eW+wYUycySePsQZaQfdIkky7olIsMTBQmSxB16D\n=lXuh\n-----END PGP MESSAGE-----\n", "fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21"}]} \ No newline at end of file diff --git a/example.yaml b/example.yaml index 62ff0a2a6..77801b829 100644 --- a/example.yaml +++ b/example.yaml @@ -1,43 +1,48 @@ # The secrets below are unreadable without access to one of the sops master key -myapp1: ENC[AES256_GCM,data:Tr7oc19nc6t1m9OrUeo=,iv:1vzlPZLfy6wa14/x17P8Ix8wEGDeY0v2dIboZmmwpww=,aad:NpobRzMzpDOkqijzONm8KglltzG+aBV7BJAxtm77veo=,tag:kaYqRgGGBhXhODSSmIZwyA==,type:str] +myapp1: ENC[AES256_GCM,data:UpXlBAV263+rZdQu4BRia0qXMDhm,iv:rnp4FZeiduOpvuVINNqDFEyLXJalg/UxKBb0TwcZBQ0=,tag:Xv0iAKLeAKqTxa3C8UPHZg==,type:str] app2: db: - user: ENC[AES256_GCM,data:YNKE,iv:H9CDb4aUHBJeF2MSTKHQuOwlLxQVdx12AhT0+Dob4JQ=,aad:jlF2KvytlQIgyMpOoO/BiQbukiMwrh1j94Oys+YMgk0=,tag:NeDysIHV9CGtMAQq9i4vMg==,type:str] - password: ENC[AES256_GCM,data:p673JCgHYw==,iv:EOOeivCp/Fd80xFdMYX0QeZn6orGTK8CeckmipjKqYY=,aad:UAhi/SHK0aCzptnFkFG4dW8Vv1ASg7TDHD6lui9mmKQ=,tag:QE6uuhRx+cGInwSVdmxXzA==,type:str] + user: ENC[AES256_GCM,data:DcCb,iv:XOc8876U/AgIaG712CNrdigwQjjkuIaIYfX2H7cv49I=,tag:Vhuhu7C1anuWc9rNBwVbDw==,type:str] + password: ENC[AES256_GCM,data:/lxxfM7WVw==,iv:cR1XPolF6ur/lIJeT3lkeIeMqlGcVzrJRTD//f/JoQ4=,tag:+wrUc73/iR2kB36CNdXfFA==,type:str] # private key for secret operations in app2 key: |- - ENC[AES256_GCM,data:Ea3zTFSOlg1PDZmBa1U2dtKl3pO4nTmaFswJx41fPfq3u8O2/Bq1UVfXn2SrO13obfr6xH4zuUceCDTvW2qvphlan5ir609EXt4dE2TEEcjVKhmAHf4LMwlZVAbvTJtlsnvo/aYJH95uctjsSX5h8pBlLaTGBGYwMrZuMyRU6vdcMWyha+piJckUc9sq7fevy1TSqIxf1Usbn/0NEklWm2VSNzQ2Urqtny6EXar+xU7NfYSRJ3mqmcJZ14oIeXPdpk962RwMEFWdYrbE7D59kWU2BgMjDxYJD5KXpWiw2YCrA/wsATxVCbZlwqC+TJFA5WAUZX756mFhV/t2Li3zQyDNUe6KkMXV9qwf/oV1j5sVRVFsKDYIBqhi3qWBVA+SO9RloQMjhru+IsdbQcS4LKq/1DrBENeZuJ0djUAxKLVfJzMGUf89ju3m9IEPovW8mfF0RbfAGRwFHMO9nEXCxrTLERf3owdR3u4j5/rNBpIvvy1z+2dy6sAx/eyNdS+cn5qO9BPAxsXpSwkaI96rlBagwH1Pfxus0x/D00j93OpE+M8MgQ/9LA68FlCFU4OAQlvw8f7MPoxnq+/+gFTS/qqjTR6EoUuX5NH2WY93YCC5TCbe4GOXyP0H05PbIWq55UMVLNcpAyac3gO4kL5O5U8=,iv:Dl61tsemKH0fdmNul/PmEEsRYFAh8GorR8GRupus/EM=,aad:Ft2aSYYukD1x8pMj1WvmodLjJV6waPy5FqdlImWyQKA=,tag:EPg4KpWqni/buCFjFL857A==,type:str] -integer: ENC[AES256_GCM,data:h9JvmpGYpXZAHg==,iv:zHUU/KWxgi3q+KDFv56QsEJ2e7fN+9Bm1u+JUSQffzs=,aad:LPYNegFpDJClRISzERd/IgcTPn5EOrlYYLRfzhDHVOo=,tag:EUNSGSD7Q4vtD+oAyTaCDw==,type:int] -float: ENC[AES256_GCM,data:wuPp0ku4iw==,iv:CdFwwT/MujK3nf7kbNXBwUrRJ25ecR1ne8DLK1jo0Xg=,aad:/L0DMviDGlgzitxVrTOINkWjxaG764zIc4+9TWWpXl4=,tag:ZyvBXcLsRcXjRR2DVE+gxA==,type:float] + ENC[AES256_GCM,data:78VMY0v+DvYl2LePvuDsN4SeyPQK9uBPE9L/kadbUgFF3S36OgWu4XJnDnpZfHRfsRDqlJM+mF2vze6+psRy+5o1O89FlOFpRykicV96kpToCM1TyaVzlQs0bAUWJ5S3H/pStQvU03E2Avx2KR5pxvOajPkb4orfU3nkgmnU8SJSZcLgmxC2eQgkx5ajXcQ26CLvQreczOGFtRGRo9WP3ePg93S4DcDrwjWsIqoMwS+2Pv+iAGVy+bhSleKe8WpmhtCSBpwlq9vNiykPEXktAhBugDyWgkLtTnAL3VcJ7z8xWoKIdGPBIMTgGk8Vdkuc0z6Y32bhyRvJr2xg08lZZDwyzpmP8+qSTi4gc/y4C8B8hIm6ldQMImSvdKNFs3yehxMTnensy/qT8wisX9UELaFClJZDRqgF2g7+2Oeci3c7RNa0HXt+qMjUREQ1Px4m3mi2AAvxU/bOduBmf/d0eyR/jn5I1wbDWH/IHkulfF0eAezG3PYS5BIkoN4iluPX87RBkxKjzkQmMUQCBb0pPSVjbVdTzKyhZD2gpca5rrxw3jbUU7wObfWO9EWxPdvzxxr/8Ul4rWzpohL+/2B0+SjHCOq4ZMk7wR4Y4dnZHd8gHb6U59g6ayelaYv3RCP4bHrW7dWYgfCeqKErsvGqzik=,iv:rOwVWlkdSomHVVXwBwxGLUgbsPvPBqnV7E12gcmmzlY=,tag:dliz9mVOhTA4zfBsCGVyqQ==,type:str] +number: ENC[AES256_GCM,data:uJyCZeY4/KnlJg==,iv:uTq9o812BitYNuuFPMqWAgIV6HarNErQV1JfvKb3tmU=,tag:hfNTc2iAoD40hk4lHihwTQ==,type:int] an_array: -- ENC[AES256_GCM,data:td1aAv4s4cOzSo0=,iv:ErVqte7GpQ3JfzVpVRf7pWSQZDHn6W0iAntKWFsMqio=,aad:RiYy8fKX/yVY7KRgXSOIzydT0+TwK7WGzSFSy+1GmVM=,tag:aSGLCmNZsGcBjxEGvNQRwA==,type:str] -- ENC[AES256_GCM,data:2K8C418jef8zoAY=,iv:cXE4Hwdl4ZHzAHHyyXqaIMFs0mn65JUehDdaw/aM0WI=,aad:RlAgUZUZ1DvxD9/lZQk9KOHKl4L+fYETaAdpDVekCaA=,tag:CORSBzis6Vy45dEvT/UtMg==,type:str] -- ENC[AES256_GCM,data:hbcOBbsaWmlnrpeuwLfh1ttsi8zj/pxMc1LYqhdksT/oQb80g2z0FE4QwUVb7VV+x98LAWHofVyV8Q==,iv:/sXHXde82r2FyG3Z3vC5x8zONB14RwC0GmtkiYEUNLI=,aad:BQb8l5fZzF/aa/EYnrOQvRfGUTq9QmJOAR/zmgOfYDA=,tag:fjNeg3Manjl6B2U2oflRhg==,type:str] -- ENC[AES256_GCM,data:LLHkzGobqL53ws6E2zglkA==,iv:g9z3zz4DUzJr4Cim0SVqKF736w2mZoItqbB0TcsGrQU=,aad:Odrvz0loqFdd9wKJz0ULMX/lyEQcX8WaHE59MgeXkcI=,tag:V+rV/AeZ4uEgtwGhlamTag==,type:str] +- ENC[AES256_GCM,data:kUbSpUwaGDtAcyI=,iv:94+201+1O3Sb9SgY7kc/1QWE9H1Tk1oN42A3OshBsb8=,tag:ciZwcgXAfhhrqltDWM7ZTQ==,type:str] +- ENC[AES256_GCM,data:pZWqRK5ZYpOfGHw=,iv:kSqcJz3/t78O+xDQkBDvoG6pxA1w5uzNAMlEBgnkRR8=,tag:RvniVpPKwr/o/VAvFrxqSA==,type:str] +- ENC[AES256_GCM,data:oAkO6uRAV70w+ZOSScMLzUk5rtwybj0epc2tesZkeXuNuM9R0I8l34yQ2/jjCEgLcGYtqB6+i4ljGg==,iv:bniytY7Y+oZen0T3OjBBlQMoHVx4TwnYUI5IgHbf2AY=,tag:7CRdL76zQkwbumga9kWgsg==,type:str] +- ENC[AES256_GCM,data:n5ZMVViz/IZwqRDgIljrQg==,iv:8A9X2UyYwyGsFKwoTO1MH69MCpUa31QJ3jSgde9i2Pc=,tag:vG4j9Y01cVg+oJ7peeA6PQ==,type:str] sops: + mac: ENC[AES256_GCM,data:rNiflHDZo2IayVwE4Wghr1VObZ4TDvg45qurpEK0+BCdi5KoNXrfmWuAr9O8baBNYvMTfPC4Yi1Yuu6bwk7NU0+uzKmWFZxGjC4GH9dAbyg5dTMF1WFIug+91wBoUC3lWwZ+ltwMvbfshNvGGebvoPlnONsM6NYC51u8s58/Hmk=,iv:M+XC1Yue8b7LW/DZl0NsQJaQFjBZagc7xZC2+PffoC4=,tag:MjDeR64LXMi7jX6WE8LEww==,type:str] + attention: This section contains key material that should only be modified with + extra care. See `sops -h`. kms: - - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyGdRODuYMHbA8Ozj8CARCAO7opMolPJUmBXd39Zlp0L2H9fzMKidHm1vvaF6nNFq0ClRY7FlIZmTm4JfnOebPseffiXFn9tG8cq7oi - enc_ts: 1439568549.245995 + - created_at: 1444233191.408969 + enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzV1o6Prf6blP9lLM4CARCAO+JL69F23WaIi3c2mQdHBTSUf1OdyRTq0+yFYxZCmCARmYAM3GEnTiuFMMCFP4I09TJJBBuc58/RTM0u arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e + - created_at: 1444233192.474385 + enc: CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAww2lEDZcq5evsCFcACARCAOxuzZh55fs9x7WE/ZpiRKIG85bvTWVn8wFnFozK/dT3tlPmNv5JwGiXQw1BG6e+fw31npvcGoIvzkwje + arn: arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d pgp: - fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21 + created_at: 1444233192.486551 enc: | -----BEGIN PGP MESSAGE----- Version: GnuPG v1 - hQIMA0t4uZHfl9qgAQ//ZvUMJOLUJyzKa/Uigwh1jKVhx3feHUitVjCWBfVTPgj1 - rRbaTcaF/mYi+rLdW+6kmAg1UEPoVgEBEiBvCTcHjyDzw3m0DoQwvK85nqOpEhkx - rjU1XAnKZ8LNFfIaj8Xo/L6qzE882gwOhfCPU+QmnkWdijs6dQof06DButQDTx5D - KlFvr9CgSa52/uPazZ41disho9guS06k+KrV/P2F4jrU5aB5mfP7YZY9mkVcm2bv - 9C5O9neNlXcivgWqKQjB5fmv1Z9yUFAUBNg98wjT8o5Hxz6P6hIbV3f+vn/Vu+VZ - Qo+E7g3/2ItaT89KAIVXgQdHhwJneoDBVpJ4rYz7LLbcvEyAbipKIY4Fl3Cn1ggH - 9odIZWA6FWZxHNhRVonMVHZ8Jei5NkUdpJltjDmPJpl3B+7XiWg4NS8dp860fLeL - 8nrkR0Z4nVK8DNg+7nQiOxHL9wye6ljWl7/xapJ5r+mYA6eLybsSSlxDo9/OmeON - CYo3jV8HT8amrXYVi4MyZ3LV2TTyGVPObnthYEN2lPSJmms6ei6t/xKaZtAj6779 - EzGbUP9VpTKKf5tqGcy9MeGEk2p5ed5hJGinrrt92cNIebcMBJpkLQAy++V/fKnH - Meecaoj1NThBnRguNuz73WSy2C5u/g7OoI50HJJCmoVXY+8D64tWmCZc8Ib2fprS - XgFcoR9u5yPkLZW8xASpRXfKKTbRTTjAXdYEyaYuuOW2nFWo62/d1mZsT7kY21ja - AhVVoxwsj45FCuk63bDVceAJJm+9xxufMp0gNW1GUk858VLyE8gn+uAB5zBcS5c= - =BN/t + hQIMA0t4uZHfl9qgAQ/+IzDsOY8rkovkbo/yECCO3yzdLBIueeQU9EpVNo85r6Dm + tDmBoQ0YmBNdFSQ6zO01N1S6IOrTshHZKkLlBjjDamSxZTyCvwMTHvDJUg1fV8h4 + QRllZmfeuIHqWCzAiIyEUHfKak0nJ+qGQtjJBMSHnHiTTnHGBA24WtgAAbbqcKyV + 7FFMNt3PNVe5GQnXfFwGLIDm5Pge83QN2JgYaeOaR2KAeGR7tPGi1k66qE1R8+lq + 7B5F92VQdAuC0dy2i09bZmzKbegDFl713D1vdvv1u60EFnCiaIGQs4r+Dob+EK1n + k8MTP9PwTW2LO/ShmLTJ1gOx+r0EWi+uebzPee566HYUOB0NMd5wx56vxxsWbmgo + OLA4/SWiWUz9N3ARouEccRlhCmGBepHBAzsFloz/SxP9/LmpkYr7e1AVaiVXKSGU + zQVzhb7EqBKzisDsvQXgZvu6apitz9OO3fXud7JaRfq8CFBCcn+iAnwfAup23STT + FXv+g4deuhJRaUdDLrZyWvoCES95dmDFrrfVUFzpSWAYqJjS0OKvbPY2TLJB43Ip + mepaMJ5RCPJuzT1yeG02HTTzA0Zkbdk2Iez3IJZBTRoKHfdCrOTQgudEF/v0TOAm + 2KQEfzQv/pcxvlPsR4ElOMr2yEtHHKdbNZq8dyuAMArXOSFU8SlDiXgTLqivw1XS + XgHy700JO7oO7Ii9WGV3J1L3TfAjd+uPTYqKsD67IKAJmUc5zyy+CrpL8l1OQ50b + FcjpqVkpIdddl1AUuY5NVBc0Xkhglz244o+xjIkULVLdlAFFGpv5CnXMLOOm5Bs= + =MGKv -----END PGP MESSAGE----- - created_at: 1443203323.058362 diff --git a/requirements.txt b/requirements.txt index a59fe2232..9953422ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ cryptography>=0.9.3 boto3>=1.1.3 ruamel.yaml>=0.10.7 +ordereddict>=1.1 +simplejson>=3.8 diff --git a/sops/__init__.py b/sops/__init__.py index 2c72888fd..754427097 100644 --- a/sops/__init__.py +++ b/sops/__init__.py @@ -11,7 +11,7 @@ from __future__ import print_function, unicode_literals import argparse -import json +import hashlib import os import re import subprocess @@ -19,7 +19,6 @@ import sys import tempfile import time from base64 import b64encode, b64decode -from collections import OrderedDict from socket import gethostname from textwrap import dedent @@ -28,6 +27,14 @@ import ruamel.yaml from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import Cipher, modes, algorithms +if sys.version_info[0] == 2 and sys.version_info[1] == 6: + # python2.6 needs simplejson and ordereddict + import simplejson as json + from ordereddict import OrderedDict +else: + import json + from collections import OrderedDict + DESC = """ `sops` is an encryption manager and editor for files that contains secrets. @@ -139,6 +146,10 @@ def main(): dest='show_master_keys', help="display master encryption keys in the file" "during editing (off by default).") + argparser.add_argument('--ignore-mac', action='store_true', + dest='ignore_mac', + help="ignore Message Authentication Code " + "during decryption") args = argparser.parse_args() kms_arns = "" @@ -181,12 +192,13 @@ def main(): # Encrypt mode: encrypt, display and exit key, tree = get_key(tree, need_key) - tree = walk_and_encrypt(tree, key) + tree = walk_and_encrypt(tree, key, isRoot=True) elif args.decrypt: # Decrypt mode: decrypt, display and exit key, tree = get_key(tree) - tree = walk_and_decrypt(tree, key) + tree = walk_and_decrypt(tree, key, isRoot=True, + ignoreMac=args.ignore_mac) else: # EDIT Mode: decrypt, edit, encrypt and save @@ -197,7 +209,8 @@ def main(): stash = dict() stash['sops'] = dict(tree['sops']) if existing_file: - tree = walk_and_decrypt(tree, key, stash=stash) + tree = walk_and_decrypt(tree, key, stash=stash, isRoot=True, + ignoreMac=args.ignore_mac) # hide the sops branch during editing if not args.show_master_keys: @@ -237,7 +250,7 @@ def main(): tree = load_file_into_tree(tmppath, otype, restore_sops=stash['sops']) os.remove(tmppath) - tree = walk_and_encrypt(tree, key, stash) + tree = walk_and_encrypt(tree, key, stash=stash, isRoot=True) tree = update_sops_branch(tree, key) # if we're in -e or -d mode, and not in -i mode, display to stdout @@ -266,6 +279,7 @@ def initialize_tree(path, itype, kms_arns=None, pgp_fps=None): """ Try to load the file from path in a tree, and failing that, initialize a new tree using default data """ + tree = OrderedDict() need_key = False try: existing_file = os.stat(path) @@ -288,9 +302,8 @@ def initialize_tree(path, itype, kms_arns=None, pgp_fps=None): if itype == "yaml": tree = ruamel.yaml.load(DEFAULT_YAML, ruamel.yaml.RoundTripLoader) elif itype == "json": - tree = json.loads(DEFAULT_JSON) + tree = json.loads(DEFAULT_JSON, object_pairs_hook=OrderedDict) else: - tree = dict() tree['data'] = DEFAULT_TEXT tree, need_key = verify_or_create_sops_branch(tree, kms_arns, pgp_fps) return tree, need_key, existing_file @@ -400,28 +413,49 @@ def update_sops_branch(tree, key): return tree -def walk_and_decrypt(branch, key, stash=None): +def walk_and_decrypt(branch, key, aad=b'', stash=None, digest=None, + isRoot=False, ignoreMac=False): """Walk the branch recursively and decrypt leaves.""" + if isRoot and not ignoreMac: + digest = hashlib.sha512() + for k, v in branch.items(): - if k == 'sops': + if k == 'sops' and isRoot: continue # everything under the `sops` key stays in clear nstash = dict() + aad += k.encode('utf-8') if stash: stash[k] = {'has_stash': True} nstash = stash[k] if isinstance(v, dict): - branch[k] = walk_and_decrypt(v, key, nstash) + branch[k] = walk_and_decrypt(v, key, aad=aad, stash=nstash, + digest=digest) elif isinstance(v, list): - branch[k] = walk_list_and_decrypt(v, key, nstash) + branch[k] = walk_list_and_decrypt(v, key, aad=aad, stash=nstash, + digest=digest) elif isinstance(v, ruamel.yaml.scalarstring.PreservedScalarString): - ev = decrypt(v, key, nstash) + ev = decrypt(v, key, aad=aad, stash=nstash, digest=digest) branch[k] = ruamel.yaml.scalarstring.PreservedScalarString(ev) else: - branch[k] = decrypt(v, key, nstash) + branch[k] = decrypt(v, key, aad=aad, stash=nstash, digest=digest) + + if isRoot and not ignoreMac: + # compute the hash computed on values with the one stored + # in the file. If they match, all is well. + if not ('mac' in branch['sops']): + panic("'mac' not found, unable to verify file integrity", 52) + h = digest.hexdigest().upper() + # We know the original hash is trustworthy because it is encrypted + # with the data key and authenticated using the tree keys + orig_h = decrypt(branch['sops']['mac'], key, aad=aad) + if h != orig_h: + panic("Hash verification failed!\nexpected %s\nbut got %s" % + (orig_h, h), 51) + return branch -def walk_list_and_decrypt(branch, key, stash=None): +def walk_list_and_decrypt(branch, key, aad=b'', stash=None, digest=None): """Walk a list contained in a branch and decrypts its values.""" nstash = dict() kl = [] @@ -430,17 +464,19 @@ def walk_list_and_decrypt(branch, key, stash=None): stash[i] = {'has_stash': True} nstash = stash[i] if isinstance(v, dict): - kl.append(walk_and_decrypt(v, key, nstash)) + kl.append(walk_and_decrypt(v, key, aad=aad, stash=nstash, + digest=digest)) elif isinstance(v, list): - kl.append(walk_list_and_decrypt(v, key, nstash)) + kl.append(walk_list_and_decrypt(v, key, aad=aad, stash=nstash, + digest=digest)) else: - kl.append(decrypt(v, key, nstash)) + kl.append(decrypt(v, key, aad=aad, stash=nstash, digest=digest)) return kl -def decrypt(value, key, stash=None): +def decrypt(value, key, aad=b'', stash=None, digest=None): """Return a decrypted value.""" - valre = b'^ENC\[AES256_GCM,data:(.+),iv:(.+),aad:(.+),tag:(.+)' + valre = b'^ENC\[AES256_GCM,data:(.+),iv:(.+),tag:(.+)' # extract fields using a regex if 'type:' in value: valre += b',type:(.+)' @@ -451,11 +487,10 @@ def decrypt(value, key, stash=None): return value enc_value = b64decode(res.group(1)) iv = b64decode(res.group(2)) - aad = b64decode(res.group(3)) - tag = b64decode(res.group(4)) + tag = b64decode(res.group(3)) valtype = 'str' try: - valtype = res.group(5) + valtype = res.group(4) except: None decryptor = Cipher(algorithms.AES(key), @@ -469,6 +504,8 @@ def decrypt(value, key, stash=None): stash['iv'] = iv stash['aad'] = aad stash['cleartext'] = cleartext + if digest: + digest.update(cleartext) if valtype == b'str': return cleartext.decode('utf-8') if valtype == b'int': @@ -477,28 +514,38 @@ def decrypt(value, key, stash=None): return float(cleartext.decode('utf-8')) -def walk_and_encrypt(branch, key, stash=None): +def walk_and_encrypt(branch, key, aad=b'', stash=None, + isRoot=False, digest=None): """Walk the branch recursively and encrypts its leaves.""" + if isRoot: + digest = hashlib.sha512() for k, v in branch.items(): - if k == 'sops': + if k == 'sops' and isRoot: continue # everything under the `sops` key stays in clear + aad += k.encode('utf-8') nstash = dict() if stash and k in stash: nstash = stash[k] if isinstance(v, dict): # recursively walk the tree - branch[k] = walk_and_encrypt(v, key, nstash) + branch[k] = walk_and_encrypt(v, key, aad=aad, stash=nstash, + digest=digest) elif isinstance(v, list): - branch[k] = walk_list_and_encrypt(v, key, nstash) + branch[k] = walk_list_and_encrypt(v, key, aad=aad, stash=nstash, + digest=digest) elif isinstance(v, ruamel.yaml.scalarstring.PreservedScalarString): - ev = encrypt(v, key, nstash) + ev = encrypt(v, key, aad=aad, stash=nstash, digest=digest) branch[k] = ruamel.yaml.scalarstring.PreservedScalarString(ev) else: - branch[k] = encrypt(v, key, nstash) + branch[k] = encrypt(v, key, aad=aad, stash=nstash, digest=digest) + if isRoot: + # finalize and store the message authentication code in encrypted form + h = digest.hexdigest().upper() + branch['sops']['mac'] = encrypt(h, key, aad=aad) return branch -def walk_list_and_encrypt(branch, key, stash=None): +def walk_list_and_encrypt(branch, key, aad=b'', stash=None, digest=None): """Walk a list contained in a branch and encrypts its values.""" nstash = dict() kl = [] @@ -506,15 +553,18 @@ def walk_list_and_encrypt(branch, key, stash=None): if stash and i in stash: nstash = stash[i] if isinstance(v, dict): - kl.append(walk_and_encrypt(v, key, nstash)) + kl.append(walk_and_encrypt(v, key, aad=aad, stash=nstash, + digest=digest)) elif isinstance(v, list): - kl.append(walk_list_and_encrypt(v, key, nstash)) + kl.append(walk_list_and_encrypt(v, key, aad=aad, stash=nstash, + digest=digest)) else: - kl.append(encrypt(v, key, nstash)) + kl.append(encrypt(v, key, aad=aad, stash=nstash, + digest=digest)) return kl -def encrypt(value, key, stash=None): +def encrypt(value, key, aad=b'', stash=None, digest=None): """Return an encrypted string of the value provided.""" valtype = 'str' if isinstance(value, int): @@ -522,6 +572,8 @@ def encrypt(value, key, stash=None): if isinstance(value, float): valtype = 'float' value = str(value).encode('utf-8') + if digest: + digest.update(value) # if we have a stash, and the value of cleartext has not changed, # attempt to take the IV and AAD value from the stash. # if the stash has no existing value, or the cleartext has changed, @@ -531,13 +583,14 @@ def encrypt(value, key, stash=None): aad = stash['aad'] else: iv = os.urandom(32) - aad = os.urandom(32) + if aad == b'': + aad = os.urandom(32) encryptor = Cipher(algorithms.AES(key), modes.GCM(iv), default_backend()).encryptor() encryptor.authenticate_additional_data(aad) enc_value = encryptor.update(value) + encryptor.finalize() - return "ENC[AES256_GCM,data:{value},iv:{iv},aad:{aad}," \ + return "ENC[AES256_GCM,data:{value},iv:{iv}," \ "tag:{tag},type:{valtype}]".format( value=b64encode(enc_value).decode('utf-8'), iv=b64encode(iv).decode('utf-8'), diff --git a/tests/test_sops.py b/tests/test_sops.py index 088efe79f..b3d58a15c 100644 --- a/tests/test_sops.py +++ b/tests/test_sops.py @@ -13,10 +13,14 @@ import unittest2 import mock import os import sys -from collections import OrderedDict import sops +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + if sys.version_info[0] == 2: import __builtin__ as builtins else: @@ -86,7 +90,7 @@ class TreeTest(unittest2.TestCase): "656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d" pgp_fps = "85D77543B3D624B63CEA9E6DBC17301B491B3F21," + \ "C9CAB0AF1165060DB58D6D6B2653B624D620786D" - tree = dict() + tree = OrderedDict() tree, ign = sops.verify_or_create_sops_branch(tree, kms_arns=kms_arns, pgp_fps=pgp_fps) @@ -132,8 +136,8 @@ class TreeTest(unittest2.TestCase): # TODO: # - test stash value m = mock.mock_open(read_data=sops.DEFAULT_YAML) - tree = dict() key = os.urandom(32) + tree = OrderedDict() with mock.patch.object(builtins, 'open', m): tree = sops.load_file_into_tree('path', 'yaml') crypttree = sops.walk_and_encrypt(tree, key) @@ -144,23 +148,26 @@ class TreeTest(unittest2.TestCase): def test_walk_and_encrypt_and_decrypt(self): """Test a roundtrip on the tree encryption/decryption code""" m = mock.mock_open(read_data=sops.DEFAULT_JSON) - tree = dict() key = os.urandom(32) + tree = OrderedDict() with mock.patch.object(builtins, 'open', m): tree = sops.load_file_into_tree('path', 'json') - crypttree = sops.walk_and_encrypt(dict(tree), key) - cleartree = sops.walk_and_decrypt(dict(crypttree), key) + tree['sops'] = dict() + crypttree = sops.walk_and_encrypt(OrderedDict(tree), key, isRoot=True) + cleartree = sops.walk_and_decrypt(OrderedDict(crypttree), key, isRoot=True) assert cleartree == tree def test_numbers_encrypt_and_decrypt(self): """Test encryption/decryption of numbers""" m = mock.mock_open(read_data='{"a":1234,"b":[567,890.123],"c":5.4999517527e+10}') - tree = dict() key = os.urandom(32) + tree = OrderedDict() with mock.patch.object(builtins, 'open', m): tree = sops.load_file_into_tree('path', 'json') - crypttree = sops.walk_and_encrypt(dict(tree), key) - cleartree = sops.walk_and_decrypt(dict(crypttree), key) + tree['sops'] = dict() + crypttree = sops.walk_and_encrypt(OrderedDict(tree), key, isRoot=True) + assert tree['sops']['mac'].startswith("ENC[AES256_GCM,data:") + cleartree = sops.walk_and_decrypt(OrderedDict(crypttree), key, isRoot=True) assert cleartree == tree def test_walk_list_and_encrypt(self): @@ -181,7 +188,8 @@ class TreeTest(unittest2.TestCase): """Test a roundtrip in the encryption/decryption code""" origin = "AAAAAAAA" key = os.urandom(32) - clearstr = sops.decrypt(sops.encrypt(origin, key), key) + aad = os.urandom(32) + clearstr = sops.decrypt(sops.encrypt(origin, key, aad=aad), key, aad=aad) assert clearstr == origin # Test keys management