From df16173611c5e5088677b50506da41bdfa99d260 Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Tue, 24 Nov 2015 09:23:01 -0500 Subject: [PATCH] Support adding & removing master keys, fixes #26 --- README.rst | 40 ++++++++++++++------ example.yaml | 36 +++++++++--------- sops/__init__.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_sops.py | 86 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 225 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 07deeb8ff..8d127fef7 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ Your AWS credentials must be present in `~/.aws/credentials`. sops uses boto3. .. code:: - $ cat ~/.aws/credentials + $ cat ~/.aws/credentials [default] aws_access_key_id = AKI..... aws_secret_access_key = mw...... @@ -156,15 +156,33 @@ steps, apart from the actual editing, are transparent to the user. Adding and removing keys ~~~~~~~~~~~~~~~~~~~~~~~~ -When creating a new files, `sops` uses the PGP and KMS defined in the command +When creating new files, `sops` uses the PGP and KMS defined in the command line arguments `--kms` and `--pgp`, or from the environment variables `SOPS_KMS_ARN` and `SOPS_PGP_FP`. That information is stored in the file under -the `sops` section. When editing a file, it is trivial to add or remove keys: -invoke `sops` with the flag **-s** to display the master keys while editing, and -add or remove kms or pgp keys under the sops section. +the `sops` section, such that decrypting files does not require providing those +parameters again. -For example, to add a KMS master key to a file, we would add the following -entry: +Master PGP and KMS keys can be added and removed from a `sops` file in one of +two ways: by using command line flag, or by editing the file directly. + +Command line flag `--add-kms`, `--add-pgp`, `--rm-kms` and `--rm-pgp` can be +used to add and remove keys from a file. These flags use the comma separated +syntax as the `--kms` and `--pgp` arguments when creating new files. + +.. code:: bash + + # add a new pgp key to the file while editing + $ sops --add-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 example.yaml + + # remove a pgp key from the file while editing + $ sops --rm-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 example.yaml + +Alternatively, invoking `sops` with the flag **-s** will display the master keys +while editing. This method can be used to add or remove kms or pgp keys under the +sops section. + +For example, to add a KMS master key to a file, add the following entry while +editing: .. code:: yaml @@ -323,10 +341,10 @@ In-place encryption/decryption also works on binary files. $ sha512sum /tmp/somerandom 9589bb20280e9d381f7a192000498c994e921b3cdb11d2ef5a986578dc2239a340b25ef30691bac72bdb14028270828dad7e8bd31e274af9828c40d216e60cbe /tmp/somerandom - $ sops -e -i /tmp/somerandom + $ sops -e -i /tmp/somerandom please wait while a data encryption key is being generated and stored securely - $ sops -d -i /tmp/somerandom + $ sops -d -i /tmp/somerandom $ sha512sum /tmp/somerandom 9589bb20280e9d381f7a192000498c994e921b3cdb11d2ef5a986578dc2239a340b25ef30691bac72bdb14028270828dad7e8bd31e274af9828c40d216e60cbe /tmp/somerandom @@ -550,7 +568,7 @@ systems. Not unlike many other organizations that operate sufficiently complex automation, we found this to be a hard problem with a number of prerequisites: 1. Secrets must be stored in YAML files for easy integration into hiera - + 2. Secrets must be stored in GIT, and when a new CloudFormation stack is built, the current HEAD is pinned to the stack. (This allows secrets to be changed in GIT without impacting the current stack that may @@ -611,7 +629,7 @@ The security of the data stored using sops is as strong as the weakest cryptographic mechanism. Values are encrypted using AES256_GCM which is the strongest symetric encryption algorithm known today. Data keys are encrypted in either KMS, which also uses AES256_GCM, or PGP which uses either RSA or -ECDSA keys. +ECDSA keys. Going from the most likely to the least likely, the threats are as follows: diff --git a/example.yaml b/example.yaml index 2191c4209..843ffb3e0 100644 --- a/example.yaml +++ b/example.yaml @@ -22,8 +22,8 @@ this: nested: value: ENC[AES256_GCM,data:TzfuYK7BOwJlmlxydTmtPKlfIvSxoaIMiqrt,iv:q+YKcwFOImx8VX4Ti1ECjBWLz32gtkxzBDq12uOsmvk=,tag:GXz+BkXKbblwfEc/dZLgzg==,type:str] sops: - mac: ENC[AES256_GCM,data:gj86GUajphvwhmUS5Z+1nK+yxqleOTOSj71WVl48K4P6R3o/K9rE+doLFl1z2xAw0TxSFnNAR5L/fpvR/3uZCQRfXb9yLTZNOAOtc0JYh3B9Epsa78uznGXnQwuuiX4rXprsrLZ0d07cEuCkhFJww7v27C6zlP8MpthpYTWMXfM=,iv:NlJqEdMb5Y6V54hbzTAwDZ067xFUvxobX05Xa2PuwZs=,tag:Wk9061XNFX8Xpp7duxE+tg==,type:str] - version: 0.90000000000000002 + mac: ENC[AES256_GCM,data:svdUk+7ahpTaWBUdXqgEy5+K6uMm210Jrm3fPvsx2VaCiONv5QIDQbUipRFOpGKubKfhJk9XPcr+4MaE6oUxW8snxkN0p1BMAqpZhQ31xdwila318TckJltgPQQfAl59CNsLf1EgweBTWhvZL5sWGOEMXfMAHuHWzN4v1CmAU3w=,iv:vyFzhy4LwFQ6pNJulze9BBt9sfIfLwhhmlrIAroO+JE=,tag:AihY+oO3J7mon003SHYrfQ==,type:str] + version: 1.0 kms: - created_at: '2015-10-25T12:52:27Z' enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAykG26ZbESEOy9KtoQCARCAO4cK6asAUiZBDmIgWk98BTvxUkvUmXYF2dxkP+Pr6F+r2oO7jhyB/FqyV5WAHCmdljs6DzBvB0FSKgdL @@ -46,26 +46,26 @@ sops: =YXAh -----END PGP MESSAGE----- - fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21 - created_at: '2015-10-25T12:52:27Z' + created_at: '2015-11-24T14:19:08Z' enc: | -----BEGIN PGP MESSAGE----- Version: GnuPG v1 - hQIMA0t4uZHfl9qgARAAjQrSvwlR66cwzHM9HkzvKcfXxy71mBwCjVYR/dazz+Bg - WWyAOsaZ9lPnR1K7ANaiKPtsF6+drEWokUsHdDc4waYMYX4Ha7kjXr9CfbrlhM6Y - gI0PrRI17Un85HjeHQYp/Vndw8c4ZKV0tOKKGGiWA+GAXiM+fDrSJBSt8wy6SJtY - t+T1Wl/VEmyvLGM9VGK+MI6Htyy2FCH0kWQ8wBA9iJv59MvBTR2s3FhdGovosk7k - 1PIRV3M5A7yjOMgHkvdC149BfqBLGcUYM+1xwXLJOGX04eCD8Y/XT41NRMH44rfq - ev6LEVJlqi50DhagkBdPp/FTpFLhhTRfkIISz236XPzZC/zDXLBSQt5DbwqymsVy - WTavSqDmOQFX2Zir+nlZcKwwCsY3funZm9jVefuQmphN+yXRM2VkK67goH5ZMeGI - uFU3xzuhyibYH/YgJT8g0fTYeiaKzcIicwN4klkhpnckrHSa8brMTYK1eZ+olB29 - XnbCM6unDnaKDJGarD9reDQt91lRsENUkj83mOrHdtGQigV2fbv3+KtfXvW4Q8Fq - lcq5oLRTch6RAcSbQfTL9fK6AjPbWZX97JjAZbeSY+HI6YRGL+Iaf26sHbZQTcuS - rrx0vX2rvkQSwdZ+ZfKC9az2/9hPjWkDLyhu5WE3KIq/SlsDl8pTLXzarGeEPN7S - XgHWXXf18MxU482uhGAysV50jpmnJXQk4SCM8QHZMqgKIDmJD4E6hq6WqEi1AR2C - 8HwuI2CMQ9skRKtoQJUV1gdSXuLYWzfJKCv0nrLk6Ot94QQV9RsxMeKaqf2V47o= - =ca1h + hQIMA0t4uZHfl9qgAQ/+LGpXd7Vn5RYK52Kpp1FPqUHiu3Yt1XylFSXT4BzHSxa9 + PR4sz83rDkeRwfitFf4ll1MB7zhCiekTUvBWUkdSZHLj9o+XX/7E3OvZ+B7HFG90 + +MLb7h2Cp8KRgEHBppxtbkNOmzgbDZ0s9vqxm+JU3IJzqmq1Roc2P4FVYtUgQxlY + spYxWzhizLxO/TJlERA8YV921vbKHhIP4I4KoWUk8gYR31b1kakrRcKI8/SDmbe8 + 6TlaPIZDxuSzY+toaesClJSkv7pMCByzyVgXbdgtHMReU8y4MSEjvhBcxrsZ3X8m + awJpw45DuZl+xhPGgFik/ERHewxMnoRUjmxHgfWLVkR8uP+FHXjbMiIAk0wMekGC + UbzbpESh5zLa2zlCgPshgYnEJobua0BrC+x3pVV8RFRrkKJXL1NuZ1gX4oFHePuV + O9UiVrsi5wpL2+jkAKqs/buDeOH4piWFoD03NAXX1SbHOHg6W/ji5C5Hen1WloXN + +NRffmmujiPFM2FzZWKiWDZgP+VopA3IHFUYv/TeepZUa3ldsaYh8tr1UAZswvG2 + lG5HGs7yMw1IPATWfpxhe0f2vzSKGc13Y+y0YEWqd6FH6ixo97XOMvVmRvl550o/ + iDN8v5xHUG7tKTNAB5aU7IFldUjwUcCqky7e4twNRJkcid6oXWBpBcgbcJLPvVnS + XgHYdVzf3Rvb+Jb5UlNs33wH5j1EHp7MVEIs1Gp8fySlW8m7Ui6lagPfyqb0n3nD + OujQzMRAaTP2iWEAQICXW1gnnDDyg2o5+WnnVVzX+YVYxCTLq66wnvB2M/yAcuM= + =ulPs -----END PGP MESSAGE----- - lastmodified: '2015-10-26T18:15:24Z' + lastmodified: '2015-11-24T14:19:08Z' attention: This section contains key material that should only be modified with extra care. See `sops -h`. diff --git a/sops/__init__.py b/sops/__init__.py index c57a50806..8812d5fdb 100644 --- a/sops/__init__.py +++ b/sops/__init__.py @@ -136,6 +136,21 @@ def main(): dest='show_master_keys', help="display master encryption keys in the file " "during editing (off by default).") + argparser.add_argument('--add-kms', dest='add_kms', + help="Add the given comma separated KMS ARNs to the" + " list of master keys on an existing file.") + argparser.add_argument('--rm-kms', dest='rm_kms', + help="Remove the given comma separated KMS ARNs " + "from the list of master keys on an existing " + "file.") + argparser.add_argument('--add-pgp', dest='add_pgp', + help="Add the given comma separated PGP fingerprint" + " to the list of master keys on an existing " + "file.") + argparser.add_argument('--rm-pgp', dest='rm_pgp', + help="Remove the given comma separated PGP " + "fingerprint from the list of master keys on " + "an existing file.") argparser.add_argument('--ignore-mac', action='store_true', dest='ignore_mac', help="ignore Message Authentication Code " @@ -175,6 +190,11 @@ def main(): kms_arns=kms_arns, pgp_fps=pgp_fps) if not existing_file: + if len(args.add_kms) > 0 or len(args.add_pgp) > 0 \ + or len(args.rm_kms) > 0 or len(args.rm_pgp) > 0: + panic("cannot add or remove keys on non-existent files, use " + "`--kms` and `--pgp` instead.", error_code=49) + # encrypt and decrypt modes are not available on non-existent files if (args.encrypt or args.decrypt): panic("cannot operate on non-existent file", error_code=100) else: @@ -278,6 +298,8 @@ def main(): error_code=200) tree = walk_and_encrypt(tree, key, stash=stash) + tree = add_new_master_keys(tree, args.add_kms, args.add_pgp) + tree = remove_master_keys(tree, args.rm_kms, args.rm_pgp) tree = update_master_keys(tree, key) os.remove(tmppath) @@ -499,6 +521,76 @@ def check_master_keys(tree): return False +def add_new_master_keys(tree, new_kms, new_pgp): + """ Add new master keys by creating a new tree and updating + the main tree with them + """ + if new_kms and len(new_kms) > 0: + newtree = {} + newtree['sops'] = {} + newtree, throwaway = parse_kms_arn(newtree, new_kms) + if 'kms' in newtree['sops']: + for newentry in newtree['sops']['kms']: + if 'kms' not in tree['sops']: + tree['sops']['kms'] = [newentry] + continue + shouldadd = True + for entry in tree['sops']['kms']: + if newentry['arn'] == entry['arn']: + # arn already present, don't re-add it + shouldadd = False + break + if shouldadd: + tree['sops']['kms'].append(newentry) + if new_pgp and len(new_pgp) > 0: + newtree = {} + newtree['sops'] = {} + newtree, throwaway = parse_pgp_fp(newtree, new_pgp) + if 'pgp' in newtree['sops']: + for newentry in newtree['sops']['pgp']: + if 'pgp' not in tree['sops']: + tree['sops']['pgp'] = [newentry] + continue + shouldadd = True + for entry in tree['sops']['pgp']: + if newentry['fp'] == entry['fp']: + # arn already present, don't re-add it + shouldadd = False + break + if shouldadd: + tree['sops']['pgp'].append(newentry) + return tree + + +def remove_master_keys(tree, rm_kms, rm_pgp): + """ remove master keys by creating a new tree and removing + the master keys present in the new tree from the old tree + """ + if rm_kms and len(rm_kms) > 0: + newtree = {} + newtree['sops'] = {} + newtree, throwaway = parse_kms_arn(newtree, rm_kms) + if 'kms' in newtree['sops'] and 'kms' in tree['sops']: + for rmentry in newtree['sops']['kms']: + i = 0 + for entry in tree['sops']['kms']: + if rmentry['arn'] == entry['arn']: + del tree['sops']['kms'][i] + i += 1 + if rm_pgp and len(rm_pgp) > 0: + newtree = {} + newtree['sops'] = {} + newtree, throwaway = parse_pgp_fp(newtree, rm_pgp) + if 'pgp' in newtree['sops'] and 'pgp' in tree['sops']: + for rmentry in newtree['sops']['pgp']: + i = 0 + for entry in tree['sops']['pgp']: + if rmentry['fp'] == entry['fp']: + del tree['sops']['pgp'][i] + i += 1 + return tree + + def walk_and_decrypt(branch, key, aad=b'', stash=None, digest=None, isRoot=True, ignoreMac=False): """Walk the branch recursively and decrypt leaves.""" diff --git a/tests/test_sops.py b/tests/test_sops.py index 10aed1792..9ba6d3255 100644 --- a/tests/test_sops.py +++ b/tests/test_sops.py @@ -197,6 +197,92 @@ class TreeTest(unittest2.TestCase): clearstr = sops.decrypt(sops.encrypt(origin, key, aad=aad), key, aad=aad) assert clearstr == origin + def test_add_kms_master_keys(self): + """ test adding a kms master key to an existing tree """ + tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } } + newkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz' + assert len(tree['sops']['kms']) == 1 + tree = sops.add_new_master_keys(tree, newkms, '') + assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' + assert tree['sops']['kms'][1]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac' + assert tree['sops']['kms'][1]['role'] == 'arn:aws:iam::927034868273:role/sops-dev-xyz' + + def test_add_pgp_master_keys_where_none_existed(self): + """ test adding a pgp master key to an existing tree + that does not have any pgp master key yet + """ + tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } } + newpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + tree = sops.add_new_master_keys(tree, '', newpgp) + assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' + assert tree['sops']['pgp'][0]['fp'] == 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + + def test_add_pgp_master_keys(self): + """ test adding a pgp master key to an existing tree """ + tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } } + newpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + assert len(tree['sops']['pgp']) == 1 + tree = sops.add_new_master_keys(tree, '', newpgp) + assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' + assert tree['sops']['pgp'][1]['fp'] == 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + + def test_add_kms_master_keys_where_none_existed(self): + """ test adding a kms master key to an existing tree + that does not have any kms master key yet + """ + tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } } + newkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz' + tree = sops.add_new_master_keys(tree, newkms, '') + assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' + assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac' + assert tree['sops']['kms'][0]['role'] == 'arn:aws:iam::927034868273:role/sops-dev-xyz' + + def test_rm_kms_master_keys(self): + """ test removing a kms master key to an existing tree """ + tree = {'sops': { 'kms': [ + {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' }, + {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac' } + ] } } + rmkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz' + assert len(tree['sops']['kms']) == 2 + tree = sops.remove_master_keys(tree, rmkms, '') + assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' + assert len(tree['sops']['kms']) == 1 + + def test_rm_pgp_master_keys_where_none_existed(self): + """ test removing a pgp master key to an existing tree + that does not have any pgp master key yet + """ + tree = {'sops': { 'kms': [ {'arn': 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' } ] } } + rmpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + tree = sops.remove_master_keys(tree, '', rmpgp) + assert tree['sops']['kms'][0]['arn'] == 'arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e' + assert len(tree['sops']['kms']) == 1 + assert 'pgp' not in tree['sops'] + + def test_rm_pgp_master_keys(self): + """ test removing a pgp master key to an existing tree """ + tree = {'sops': { 'pgp': [ + {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' }, + {'fp': 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' } + ] } } + rmpgp = 'E60892BB9BD89A69F759A1A0A3D652173B763E8F' + assert len(tree['sops']['pgp']) == 2 + tree = sops.remove_master_keys(tree, '', rmpgp) + assert len(tree['sops']['pgp']) == 1 + assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' + + def test_rm_kms_master_keys_where_none_existed(self): + """ test removing a kms master key to an existing tree + that does not have any kms master key yet + """ + tree = {'sops': { 'pgp': [ {'fp': '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' } ] } } + rmkms = 'arn:aws:kms:us-east-1:656532927350:key/920abb2e-c2b3-9090-943a-047fa387f3ac+arn:aws:iam::927034868273:role/sops-dev-xyz' + tree = sops.remove_master_keys(tree, rmkms, '') + assert tree['sops']['pgp'][0]['fp'] == '1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A' + assert len(tree['sops']['pgp']) == 1 + assert 'kms' not in tree['sops'] + # Test keys management def test_get_key(self): """Test we obtain a 256 bits symetric key."""