getsops/sops
1
0
Fork 0
mirror of https://github.com/getsops/sops.git synced 2026-02-05 12:45:21 +01:00
Code Issues Activity

Add support for encrypting binary files, treat all text as bytes, fixes #22

Browse Source
This commit is contained in:
Julien Vehent
2015-10-26 14:17:38 -04:00
parent 5e1e67ecb4
commit ec34a75816
6 changed files with 228 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
Makefile
View File

@@ -43,7 +43,7 @@ functional-tests:
echo "Testing Python$$ver $$type decryption" && \
python$$ver sops/__init__.py -d example.$$type > /tmp/testdata.$$type && \
echo "Testing Python$$ver $$type encryption" && \
python$$ver sops/__init__.py -e /tmp/testdata.$$type > /tmp/testdata$$ver.$$type; \
python$$ver sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata.$$type > /tmp/testdata$$ver.$$type; \
done && \
echo "Testing Python2.6 decryption of a 2.7 $$type file" && \
python2.6 sops/__init__.py -d /tmp/testdata2.7.$$type > /dev/null && \
@@ -57,6 +57,17 @@ functional-tests:
python3.4 sops/__init__.py -d /tmp/testdata2.6.$$type > /dev/null && \
echo "Testing Python3.4 decryption of a 2.7 $$type file" && \
python3.4 sops/__init__.py -d /tmp/testdata2.7.$$type > /dev/null || exit 1; \
done && \
for ver in 2.6 2.7 3.4; do \
echo "Testing Python$$ver round-trip on binary file" && \
dd if=/dev/urandom of=/tmp/testdata-$$ver-randomfile bs=1024 count=1024 2>&1 1>/dev/null && \
python$$ver sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata-$$ver-randomfile > /tmp/testdata-$$ver-randomfile.enc && \
python$$ver sops/__init__.py -d /tmp/testdata-$$ver-randomfile.enc > /tmp/testdata-$$ver-randomfile.dec && \
if [ $$(sha256sum /tmp/testdata-$$ver-randomfile | cut -d ' ' -f 1) != $$(sha256sum /tmp/testdata-$$ver-randomfile.dec | cut -d ' ' -f 1) ]; then \
echo "Binary file roundtrip failed, checksum doesn't match"; exit 0; \
else \
echo "Binary file roundtrip succeeded"; \
fi; \
done
functional-tests-once:
@@ -65,7 +76,7 @@ functional-tests-once:
echo "Testing $$type decryption" && \
python sops/__init__.py -d example.$$type > /tmp/testdata.$$type && \
echo "Testing $$type encryption" && \
python sops/__init__.py -e /tmp/testdata.$$type > /tmp/testdataenc.$$type; \
python sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata.$$type > /tmp/testdataenc.$$type; \
echo "Testing $$type re-decryption" && \
python sops/__init__.py -d /tmp/testdataenc.$$type > /dev/null || exit 1; \
done

4
example.json
View File

@@ -19,7 +19,7 @@
}
],
"sops": {
"mac": "ENC[AES256_GCM,data:IIVPhOc9mNHBL4tYpyBlgi1EgpC3UUd/ndLkT4ZDvn4RmScQzUVkWwblylqR26ObR1U3sTmmQLnE5w/eLugssS+SycPzIWqz7wRWDCbZxrU3wLVV1Qa7BzZDVarKs94FbY+46+9NLL+/M4QcGgfN8aArDj4N3NrCCfazyGSEKuc=,iv:6ukma4Rj0e2027T/S2JWKcqvJwSwT3pDMmJfSGbvek0=,tag:Ek6gppFf6TaaV9J9pIUlKA==,type:str]",
"mac": "ENC[AES256_GCM,data:DA4H2c++XgR3Oy8LThU70sIGfd11jdy+7vSs0b2n1bEWd6XgCkCEJWDpy58CJPISJY/y8iOCyEh/oeOaD0/Y+qqoOxAeK/FQmXDpKRQrrSxPcn3nbAMbkaTvTaPwjrgtJXH2cN0lQwlPF+s9FifZXKJe0aGwj7jbxjfWXaEc6pk=,iv:iwbQBuoEaEWLhA1TYeuoxFoO1ITHGg2qeciD4aUA2pw=,tag:oz9bhv60yOiQ8IxurJw0Zw==,type:str]",
"version": 0.90000000000000002,
"kms": [
{
@@ -45,7 +45,7 @@
"enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgARAAgQdMpnTNMCdbdFRpBsC9kxi334LbBrFUkp5lI+YzutZy\nSic85ea06FGL3O93tII9mwGAsESwKlN4nX0d31vuh/lYxMDakyd1IK/BkMG4Z1xG\n52MsACG/pyitMBXkIIyjmR0tVR+CixDsy5cUJxoWq+mfuE2ywziPY+KbEZ50hFXg\naAdKCdInXlLHdId+aXhThhXUGN1seQjtdyZjVXnp8c9hHS2YQdyp/SZf47NJ4A2y\nkO40kNS4oaHUUZIZLtzaFhWytZlpWEJJkIgH/vefL3jLW4SiIiqz24wr7MncsF+A\np8Pteulc5VrvA5CzQIq9qF3Zwn9HV2a0KWLZ/J29EYzSM8u9HLOYqsmNKt0TcVbX\n6eoG3JTJoRDrzO0DZvR3pMm4gQ0WXzHKzpu8g+JYnoQ19AMWJAPbTp5ej3MWHcXD\nXFjz4gsSYbwc4h/zVBOWsYoHlyTLUMwg2BA1YiL89xs8MIhIHOAmvM0mv+QuZQ7S\nCfc1mS04CZSmJvTcNkvE5n76n2iXs6nYNk8TYyQlhYebuQmJQKJuUYjKIHhuxZFa\n30WaSGnKHqIQn1pl7jqyqm8sVTzaKMyhbM0T+UQUJhXcWVr7r+CtRAt8XjVnJMvo\nviJwTWy1Ddo0Vu1licMFJXMnQbQlVh+CZS6FHqcbxfPaYfe7JldGmhwKg+F/NEHS\nXgEf78iLm3FNb4yeOkB/z2xjiZ3XvUAQjsUK5ofF1CJYcQ//YIFex1oO55Z0+qIt\njdDtqivLgf4SFRf0uhOxUrQNuFAvY361F1mvrGPcTubh/Ygq0aVzWzgC9gn7DTo=\n=uQw0\n-----END PGP MESSAGE-----\n"
}
],
"lastmodified": "2015-10-25T13:54:31Z",
"lastmodified": "2015-10-26T18:15:31Z",
"attention": "This section contains key material that should only be modified with extra care. See `sops -h`."
}
}

35
example.txt
View File

@@ -1,2 +1,33 @@
ENC[AES256_GCM,data:Q0wnzAzEA+eYHxJcLu84qY6HWU6U8WD9jj5sgnx18oTvXGOJs42mLn0UUe944sB4rROV6BK6vL11YnIad5aNE9O7UC/DNsbQklB3p73olHCwCmIPErQch0Ir8gGlsQjXYb3isWgIl1jee9bFvPtNKdVojYV6clTaSGfBtAC67wC9TRT935AbzOlpdfa1G5jQiq89zHNebytGZtU=,iv:FdFKnnAlai/yZi8/O/eFNtaBWQGdETjTuVByAQ21xO0=,tag:WxJoPC4YGoMCx3QbrSvJ7Q==,type:str]
SOPS={"attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", "kms": [{"arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e", "created_at": 1444233233.6924219, "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxatks17s0ZWQIyPi8CARCAO65vxmVs4SOASbNDdnwdeOlg75rz7oeqWId2JyQU8sNyz7+TNvvsLIjIR50AGMwnbMIgTmbM99LDi6Vo"}, {"arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d", "created_at": 1444233235.129884, "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzI3YJKPROE+fG2vJYCARCAO2IeX+3IeMkOOOsQauVrUTP9FVFmcmpXYDT41PDt8nhFvU/Q9RUUoVG1OLeWK+KBDZgu1NWGeUTN3TTs"}], "lastmodified": "2015-10-25T13:55:01Z", "mac": "ENC[AES256_GCM,data:vh99DgltlYdUECiiK/XW5JnBaZKX43Eb0RJ4Xc7KITVU0LCkfaSA9kgIfw4zWu6ieo1ENFlWrxx8iM04gROoalMcA/+VIs/yQTacpA19/oWmKSdN3bHW6lLTOVAWEQWIO7gjnrYuWA8fzSP4PopiMDE6unVJoC4NhrohgTxdQps=,iv:09HRNaX95n0JD6Avo8DoXBJJGuBWx7lMjK9y+icgeJA=,tag:UAAP1+A0ZNBDKbug5ZQHhw==,type:str]", "pgp": [{"created_at": "2015-10-08T15:32:06Z", "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhIwDEEVDpnzXnMABA/4uvvk2EDmAkmHKu4RMTq/NGSK7ZXuY7QATPdT+M0lkQGV4\nVmHlVVXe/y2qr5ouI+k3In3Fk7HR5yFDH5G2Jz3PwuosLVw3M2XmNXZ8bvcRcvKB\nI+6WNGOC2M1bvVeqTETL77nyd5fRuhDFVjQtf/oYym6IGiX9S1UH0Mx3rkDNyNJc\nAfVY+u3DNvLI5VDXMms/XQOkwEYiCL93QnWgGbSVxDXPRp3rDXTeoWEzZNXadJ6E\nKNEnToUnVXrjOH6YwHsjDc6p6djaONxlKhy+kEdoM/+AX04ukdgvyacUfbg=\n=kYWb\n-----END PGP MESSAGE-----\n", "fp": "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A"}, {"created_at": 1444233235.1358621, "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgAQ/9GNa5B4AkO7UODicvjpsgEGLd++1mJteKOwww/08src+H\nnfe/VtTvOdCNVNwvkeKtANvM5DCX9RVTjul4SH7iKd/O9XmTFXA66fhgAbRmEczm\npzQXog/res0u/q+mVwdSDqx/6qBViIcz1Zgc5oFnAneRlAke2/UsNFuFbtaQDZZh\nuralZFdrLx/DWjqEWXEh9D+caek2z/Tjhl/PQ6JNPEa7aZfMLjuTuaoPkSgd87Zc\ndnz/UL77Wx1zdv/cLtO2XvJhOvi0BF9dkg4evouTtNJs+WjQvkBCAijwdC5JdjTz\nWj4mV4H/YdlOn+j2ng3GGmF6GIX5x9FLLD5a9PjSgHVvAH8ZpXkCVY2U8e7QAW7+\nv3KLKGZFWvke62pmypj3777Z5MBj/SJAlzmuPdCLQCXIIpozqK4N4qTvg4Rt5TsN\n8YH9HYfWhX6fHvd67alwrz4IV3g1LgCKCGQd0EXl8pjYwErspGym3UOyZKSD4dDb\nH8zdbr2bQxZ2dJR3o+DVTdohfFjxUqHAZ8bO3vkUT4xblY8n2NnIUWxw3tDHdV/6\niXWVfRcgsIRmFM8qZ7CwwxDZFgLGY3oPhzNmze+B1g5xMG/l4MbKwjCb2EQ38CDr\nDG11GMG5ewhZUDwry4aDpxQMUhvuLBupve+caHzs62zTyWxurwLwfzOHbUyCxbjS\nXAEQ++zCoKncWsAxJdaoIvAvTJBEJeRyGToPESe8iYjmkT1jYZCMj30opOmOZ94M\nE0X4OYpb8FGL/QhOASMe8eW+wYUycySePsQZaQfdIkky7olIsMTBQmSxB16D\n=lXuh\n-----END PGP MESSAGE-----\n", "fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21"}], "version": 0.90000000000000002}
{
"data": "ENC[AES256_GCM,data:Q0wnzAzEA+eYHxJcLu84qY6HWU6U8WD9jj5sgnx18oTvXGOJs42mLn0UUe944sB4rROV6BK6vL11YnIad5aNE9O7UC/DNsbQklB3p73olHCwCmIPErQch0Ir8gGlsQjXYb3isWgIl1jee9bFvPtNKdVojYV6clTaSGfBtAC67wC9TRT935AbzOlpdfa1G5jQiq89zHNebytGZtU=,iv:FdFKnnAlai/yZi8/O/eFNtaBWQGdETjTuVByAQ21xO0=,tag:WxJoPC4YGoMCx3QbrSvJ7Q==,type:str]",
"sops": {
"mac": "ENC[AES256_GCM,data:e68r888d6QXxhzr/uKSV6zsJvYkS+zh/YNvY+8+j6t5WvGEseMFfALRlSDT01TzsQt/nq3qb04QViEqr8+uEA0YLJb8Dkz9+3LyAZCdC/RL0oAvteW7R1n/ULRaz6O2ScwIWsvHE3uVnLJzZweZOm9BX4ZIewRm+Ua3FqT/ugY8=,iv:tTHQ8bW8odQwxgwCTBOZsGf9IJR3xPQMJVb3XiYEw08=,tag:L7vGe0dJ+W/PBnbIXupOzA==,type:str]",
"version": 0.90000000000000002,
"kms": [
{
"arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e",
"created_at": 1444233233.6924219,
"enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxatks17s0ZWQIyPi8CARCAO65vxmVs4SOASbNDdnwdeOlg75rz7oeqWId2JyQU8sNyz7+TNvvsLIjIR50AGMwnbMIgTmbM99LDi6Vo"
},
{
"arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d",
"created_at": 1444233235.129884,
"enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzI3YJKPROE+fG2vJYCARCAO2IeX+3IeMkOOOsQauVrUTP9FVFmcmpXYDT41PDt8nhFvU/Q9RUUoVG1OLeWK+KBDZgu1NWGeUTN3TTs"
}
],
"pgp": [
{
"created_at": "2015-10-08T15:32:06Z",
"enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhIwDEEVDpnzXnMABA/4uvvk2EDmAkmHKu4RMTq/NGSK7ZXuY7QATPdT+M0lkQGV4\nVmHlVVXe/y2qr5ouI+k3In3Fk7HR5yFDH5G2Jz3PwuosLVw3M2XmNXZ8bvcRcvKB\nI+6WNGOC2M1bvVeqTETL77nyd5fRuhDFVjQtf/oYym6IGiX9S1UH0Mx3rkDNyNJc\nAfVY+u3DNvLI5VDXMms/XQOkwEYiCL93QnWgGbSVxDXPRp3rDXTeoWEzZNXadJ6E\nKNEnToUnVXrjOH6YwHsjDc6p6djaONxlKhy+kEdoM/+AX04ukdgvyacUfbg=\n=kYWb\n-----END PGP MESSAGE-----\n",
"fp": "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A"
},
{
"created_at": 1444233235.1358621,
"enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA0t4uZHfl9qgAQ/9GNa5B4AkO7UODicvjpsgEGLd++1mJteKOwww/08src+H\nnfe/VtTvOdCNVNwvkeKtANvM5DCX9RVTjul4SH7iKd/O9XmTFXA66fhgAbRmEczm\npzQXog/res0u/q+mVwdSDqx/6qBViIcz1Zgc5oFnAneRlAke2/UsNFuFbtaQDZZh\nuralZFdrLx/DWjqEWXEh9D+caek2z/Tjhl/PQ6JNPEa7aZfMLjuTuaoPkSgd87Zc\ndnz/UL77Wx1zdv/cLtO2XvJhOvi0BF9dkg4evouTtNJs+WjQvkBCAijwdC5JdjTz\nWj4mV4H/YdlOn+j2ng3GGmF6GIX5x9FLLD5a9PjSgHVvAH8ZpXkCVY2U8e7QAW7+\nv3KLKGZFWvke62pmypj3777Z5MBj/SJAlzmuPdCLQCXIIpozqK4N4qTvg4Rt5TsN\n8YH9HYfWhX6fHvd67alwrz4IV3g1LgCKCGQd0EXl8pjYwErspGym3UOyZKSD4dDb\nH8zdbr2bQxZ2dJR3o+DVTdohfFjxUqHAZ8bO3vkUT4xblY8n2NnIUWxw3tDHdV/6\niXWVfRcgsIRmFM8qZ7CwwxDZFgLGY3oPhzNmze+B1g5xMG/l4MbKwjCb2EQ38CDr\nDG11GMG5ewhZUDwry4aDpxQMUhvuLBupve+caHzs62zTyWxurwLwfzOHbUyCxbjS\nXAEQ++zCoKncWsAxJdaoIvAvTJBEJeRyGToPESe8iYjmkT1jYZCMj30opOmOZ94M\nE0X4OYpb8FGL/QhOASMe8eW+wYUycySePsQZaQfdIkky7olIsMTBQmSxB16D\n=lXuh\n-----END PGP MESSAGE-----\n",
"fp": "85D77543B3D624B63CEA9E6DBC17301B491B3F21"
}
],
"lastmodified": "2015-10-26T18:17:21Z",
"attention": "This section contains key material that should only be modified with extra care. See `sops -h`."
}
}

4
example.yaml
View File

@@ -22,7 +22,7 @@ this:
nested:
value: ENC[AES256_GCM,data:TzfuYK7BOwJlmlxydTmtPKlfIvSxoaIMiqrt,iv:q+YKcwFOImx8VX4Ti1ECjBWLz32gtkxzBDq12uOsmvk=,tag:GXz+BkXKbblwfEc/dZLgzg==,type:str]
sops:
mac: ENC[AES256_GCM,data:e3y7iNcEW4XuADj02f8mqJpA1I3nNkzk2Hx2k7NjT7KAyYF0fZGwVaOYu0/nIADUp0rSknJY827W++TcRfyM2iwRQ1FH7ydLWCYZsiL8UJtC7SrTy1goAxqCvUpJX5YlgB3jZLw9XmkxSCQ/oHT6JWwyqLtVtuHV6zWUds4s5Oc=,iv:WFMmLUkiuEL3ILZaP5RRx+uPOTjHFv+FNJUR6GJvZjI=,tag:G4+LNwAXecQrJLJs1g+2BA==,type:str]
mac: ENC[AES256_GCM,data:gj86GUajphvwhmUS5Z+1nK+yxqleOTOSj71WVl48K4P6R3o/K9rE+doLFl1z2xAw0TxSFnNAR5L/fpvR/3uZCQRfXb9yLTZNOAOtc0JYh3B9Epsa78uznGXnQwuuiX4rXprsrLZ0d07cEuCkhFJww7v27C6zlP8MpthpYTWMXfM=,iv:NlJqEdMb5Y6V54hbzTAwDZ067xFUvxobX05Xa2PuwZs=,tag:Wk9061XNFX8Xpp7duxE+tg==,type:str]
version: 0.90000000000000002
kms:
- created_at: '2015-10-25T12:52:27Z'
@@ -66,6 +66,6 @@ sops:
8HwuI2CMQ9skRKtoQJUV1gdSXuLYWzfJKCv0nrLk6Ot94QQV9RsxMeKaqf2V47o=
=ca1h
-----END PGP MESSAGE-----
lastmodified: '2015-10-25T13:55:10Z'
lastmodified: '2015-10-26T18:15:24Z'
attention: This section contains key material that should only be modified with
extra care. See `sops -h`.

278
sops/__init__.py
View File

@@ -181,110 +181,102 @@ def main():
if args.encrypt:
# Encrypt mode: encrypt, display and exit
key, tree = get_key(tree, need_key)
tree = walk_and_encrypt(tree, key)
finalize_output(tree, args.file, encrypt=True, in_place=args.in_place,
output_type=otype)
elif args.decrypt:
if args.decrypt:
# Decrypt mode: decrypt, display and exit
key, tree = get_key(tree)
tree = walk_and_decrypt(tree, key, ignoreMac=args.ignore_mac)
else:
# EDIT Mode: decrypt, edit, encrypt and save
key, tree = get_key(tree, need_key)
# we need a stash to save the IV and AAD and reuse them
# if a given value has not changed during editing
stash = dict()
stash['sops'] = dict(tree['sops'])
if existing_file:
tree = walk_and_decrypt(tree, key, stash=stash,
ignoreMac=args.ignore_mac)
# hide the sops branch during editing
if not args.show_master_keys:
tree.pop('sops', None)
finalize_output(tree, args.file, decrypt=True, in_place=args.in_place,
output_type=otype, tree_path=args.tree_path)
# the decrypted tree is written to a tempfile and an editor
# is opened on the file
tmppath = write_file(tree, filetype=otype)
tmpstamp = os.stat(tmppath)
print("temp file created at %s" % tmppath, file=sys.stderr)
# EDIT Mode: decrypt, edit, encrypt and save
key, tree = get_key(tree, need_key)
# open an editor on the file and, if the file is yaml or json,
# verify that it doesn't contain errors before continuing
valid_syntax = False
has_master_keys = False
while not valid_syntax or not has_master_keys:
run_editor(tmppath)
# we need a stash to save the IV and AAD and reuse them
# if a given value has not changed during editing
stash = dict()
stash['sops'] = dict(tree['sops'])
if existing_file:
tree = walk_and_decrypt(tree, key, stash=stash,
ignoreMac=args.ignore_mac)
# hide the sops branch during editing
if not args.show_master_keys:
tree.pop('sops', None)
# the decrypted tree is written to a tempfile and an editor
# is opened on the file
tmppath = write_file(tree, filetype=otype)
tmpstamp = os.stat(tmppath)
print("temp file created at %s" % tmppath, file=sys.stderr)
# open an editor on the file and, if the file is yaml or json,
# verify that it doesn't contain errors before continuing
valid_syntax = False
has_master_keys = False
while not valid_syntax or not has_master_keys:
run_editor(tmppath)
try:
valid_syntax = validate_syntax(tmppath, otype)
except Exception as e:
try:
valid_syntax = validate_syntax(tmppath, otype)
except Exception as e:
try:
print("Syntax error: %s\nPress a key to return into "
"the editor, or ctrl+c to exit without saving." % e,
file=sys.stderr)
raw_input()
except KeyboardInterrupt:
os.remove(tmppath)
panic("ctrl+c captured, exiting without saving", 85)
print("Syntax error: %s\nPress a key to return into "
"the editor, or ctrl+c to exit without saving." % e,
file=sys.stderr)
raw_input()
except KeyboardInterrupt:
os.remove(tmppath)
panic("ctrl+c captured, exiting without saving", 85)
if args.show_master_keys:
# use the sops data from the file
tree = load_file_into_tree(tmppath, otype)
else:
# sops branch was removed for editing, restoring it
tree = load_file_into_tree(tmppath, otype,
restore_sops=stash['sops'])
if check_master_keys(tree):
has_master_keys = True
else:
try:
print("Could not find a valid master key to encrypt the "
"data key with.\nAdd at least one KMS or PGP "
"master key to the `sops` branch,\nor ctrl+c to "
"exit without saving.")
raw_input()
except KeyboardInterrupt:
os.remove(tmppath)
panic("ctrl+c captured, exiting without saving", 85)
if args.show_master_keys:
# use the sops data from the file
tree = load_file_into_tree(tmppath, otype)
else:
# sops branch was removed for editing, restoring it
tree = load_file_into_tree(tmppath, otype,
restore_sops=stash['sops'])
if check_master_keys(tree):
has_master_keys = True
else:
try:
print("Could not find a valid master key to encrypt the "
"data key with.\nAdd at least one KMS or PGP "
"master key to the `sops` branch,\nor ctrl+c to "
"exit without saving.")
raw_input()
except KeyboardInterrupt:
os.remove(tmppath)
panic("ctrl+c captured, exiting without saving", 85)
# verify if file has been modified, and if not, just exit
tmpstamp2 = os.stat(tmppath)
if tmpstamp == tmpstamp2:
os.remove(tmppath)
panic("%s has not been modified, exit without writing" % args.file,
error_code=200)
tree = walk_and_encrypt(tree, key, stash=stash)
tree = update_master_keys(tree, key)
# verify if file has been modified, and if not, just exit
tmpstamp2 = os.stat(tmppath)
if tmpstamp == tmpstamp2:
os.remove(tmppath)
panic("%s has not been modified, exit without writing" % args.file,
error_code=200)
# if we're in -e or -d mode, and not in -i mode, display to stdout
if args.encrypt and not args.in_place:
write_file(tree, path='/dev/stdout', filetype=otype)
tree = walk_and_encrypt(tree, key, stash=stash)
tree = update_master_keys(tree, key)
os.remove(tmppath)
elif args.decrypt and not args.in_place:
if args.tree_path:
tree = truncate_tree(tree, args.tree_path)
write_file(tree, path='/dev/stdout', filetype=otype)
# otherwise, write the tree to a file
else:
path = write_file(tree, path=args.file, filetype=otype)
print("file written to %s" % (path), file=sys.stderr)
finalize_output(tree, args.file, output_type=otype)
def detect_filetype(file):
"""Detect the type of file based on its extension.
Return a string that describes the format: `text`, `yaml`, `json`
Return a string that describes the format: `bytes`, `yaml`, `json`
"""
base, ext = os.path.splitext(file)
if (ext == '.yaml') or (ext == '.yml'):
return 'yaml'
elif ext == '.json':
return 'json'
return 'text'
return 'bytes'
def initialize_tree(path, itype, kms_arns=None, pgp_fps=None):
@@ -299,16 +291,10 @@ def initialize_tree(path, itype, kms_arns=None, pgp_fps=None):
existing_file = False
if existing_file:
# read the encrypted file from disk
try:
tree = load_file_into_tree(path, itype)
except Exception as e:
panic("failed to load file: %s" % e, 72)
try:
tree, need_key = verify_or_create_sops_branch(tree,
kms_arns=kms_arns,
pgp_fps=pgp_fps)
except Exception as e:
panic("failed to initialize encryption data: %s" % e, 32)
tree = load_file_into_tree(path, itype)
tree, need_key = verify_or_create_sops_branch(tree,
kms_arns=kms_arns,
pgp_fps=pgp_fps)
# try to set the input version to the one set in the file
try:
global INPUT_VERSION
@@ -335,21 +321,27 @@ def load_file_into_tree(path, filetype, restore_sops=None):
"""
tree = OrderedDict()
with open(path, "rt") as fd:
with open(path, "rb") as fd:
if filetype == 'yaml':
tree = ruamel.yaml.load(fd, ruamel.yaml.RoundTripLoader)
elif filetype == 'json':
tree = json.load(fd, object_pairs_hook=OrderedDict)
data = fd.read()
if isinstance(data, bytes):
data = data.decode('utf-8')
tree = json.loads(data, object_pairs_hook=OrderedDict)
else:
for line in fd:
if line.startswith('SOPS='):
tree['sops'] = json.loads(
line.rstrip('\n').split('=', 1)[1])
else:
if 'data' not in tree:
tree['data'] = str()
tree['data'] += line
if not (restore_sops is None):
data = fd.read()
# try to guess what type of file it is. It may be a previously sops
# encrypted file, in which case it's in JSON format. If not, load
# the bytes as such in the 'data' key.
try:
tree = json.loads(data.decode('utf-8'),
object_pairs_hook=OrderedDict)
if "version" not in tree['sops']:
tree['data'] = data
except:
tree['data'] = data
if restore_sops:
tree['sops'] = restore_sops.copy()
return tree
@@ -568,15 +560,32 @@ def decrypt(value, key, aad=b'', stash=None, digest=None):
).decryptor()
decryptor.authenticate_additional_data(aad)
cleartext = decryptor.update(enc_value) + decryptor.finalize()
if stash:
# save the values for later if we need to reencrypt
stash['iv'] = iv
stash['aad'] = aad
stash['cleartext'] = cleartext
if digest:
digest.update(cleartext)
if valtype == b'bytes':
return cleartext
if valtype == b'str':
return cleartext.decode('utf-8')
# Welcome to python compatibility hell... :(
# Python 2 treats everything as str, but python 3 treats bytes and str
# as different types. So if a file was encrypted by sops with py2, and
# contains bytes data, it will have type 'str' and py3 will decode
# it as utf-8. This will result in a UnicodeDecodeError exception
# because random bytes are not unicode. So the little try block below
# catches it and returns the raw bytes if the value isn't unicode.
cv = cleartext
try:
cv = cleartext.decode('utf-8')
except UnicodeDecodeError:
return cleartext
return cv
if valtype == b'int':
return int(cleartext.decode('utf-8'))
if valtype == b'float':
@@ -585,6 +594,7 @@ def decrypt(value, key, aad=b'', stash=None, digest=None):
if cleartext.lower() == b'true':
return True
return False
panic("unknown type "+valtype, 23)
def walk_and_encrypt(branch, key, aad=b'', stash=None,
@@ -614,6 +624,7 @@ def walk_and_encrypt(branch, key, aad=b'', stash=None,
if isRoot:
branch['sops']['lastmodified'] = NOW
# finalize and store the message authentication code in encrypted form
h = str()
h = digest.hexdigest().upper()
mac = encrypt(h, key,
aad=branch['sops']['lastmodified'].encode('utf-8'))
@@ -642,16 +653,28 @@ def walk_list_and_encrypt(branch, key, aad=b'', stash=None, digest=None):
def encrypt(value, key, aad=b'', stash=None, digest=None):
"""Return an encrypted string of the value provided."""
valtype = 'str'
if isinstance(value, int):
valtype = 'int'
if isinstance(value, float):
valtype = 'float'
if isinstance(value, bool):
# save the original type
# the order in which we do this matters. For example, a bool
# is also an int, but an int isn't a bool, so we test for bool first
if isinstance(value, str) or \
(sys.version_info[0] == 2 and isinstance(value, unicode)):
valtype = 'str'
elif isinstance(value, bool):
valtype = 'bool'
value = str(value).encode('utf-8')
elif isinstance(value, int):
valtype = 'int'
elif isinstance(value, float):
valtype = 'float'
else:
valtype = 'bytes'
if not isinstance(value, bytes):
# if not bytes, convert to bytes
value = str(value).encode('utf-8')
if digest:
digest.update(value)
# if we have a stash, and the value of cleartext has not changed,
# attempt to take the IV.
# if the stash has no existing value, or the cleartext has changed,
@@ -857,6 +880,32 @@ def encrypt_key_with_pgp(key, entry):
return entry
def finalize_output(tree, path, encrypt=False, decrypt=False, in_place=False,
output_type="json", tree_path=None):
""" Write the final output of sops to a destination path """
# if we're in -e or -d mode, and not in -i mode, display to stdout
if encrypt and not in_place:
if output_type == "bytes":
output_type = "json"
write_file(tree, path='/dev/stdout', filetype=output_type)
elif decrypt and not in_place:
if tree_path:
tree = truncate_tree(tree, tree_path)
# don't show sops metadata in decrypt mode
write_file(tree, path='/dev/stdout', filetype=output_type)
# otherwise, write the tree to a file
else:
if output_type == "bytes":
output_type = "json"
path = write_file(tree, path=path, filetype=output_type)
print("file written to %s" % (path), file=sys.stderr)
# it's called "finalize" for a reason...
sys.exit(0)
def write_file(tree, path=None, filetype=None):
"""Write the tree content in a file using filetype format.
@@ -886,13 +935,14 @@ def write_file(tree, path=None, filetype=None):
fd.write(json.dumps(tree, indent=4).encode('utf-8'))
else:
if 'data' in tree:
# add a newline if there's none
if tree['data'][-1:] != '\n':
tree['data'] += '\n'
fd.write(tree['data'].encode('utf-8'))
try:
fd.write(tree['data'].encode('utf-8'))
except:
fd.write(tree['data'])
if 'sops' in tree:
jsonstr = json.dumps(tree['sops'], sort_keys=True)
fd.write(("SOPS=%s" % jsonstr).encode('utf-8'))
fd.close()
return path
@@ -919,9 +969,9 @@ def run_editor(path):
def validate_syntax(path, filetype):
"""Attempt to load a file and return an exception if it fails."""
if filetype == 'text':
if filetype == 'bytes':
return True
with open(path, "rt") as fd:
with open(path, "rb") as fd:
if filetype == 'yaml':
ruamel.yaml.load(fd, ruamel.yaml.RoundTripLoader)
if filetype == 'json':

24
tests/test_sops.py
View File

@@ -55,13 +55,6 @@ class TreeTest(unittest2.TestCase):
restore_sops=b)
assert tree['sops']['kms'][0]['arn'] == 'test'
@mock.patch('sops.json.load')
def test_example_with_a_mocked_call(self, json_mock):
m = mock.mock_open(read_data='"content"')
with mock.patch.object(builtins, 'open', m):
sops.load_file_into_tree('path', 'json')
json_mock.assert_called_with(m(), object_pairs_hook=OrderedDict)
def test_detect_filetype_handle_json(self):
assert sops.detect_filetype("file.json") == "json"
@@ -72,7 +65,7 @@ class TreeTest(unittest2.TestCase):
assert sops.detect_filetype("file.yaml") == "yaml"
def test_detect_filetype_returns_text_if_unknown(self):
assert sops.detect_filetype("file.xml") == "text"
assert sops.detect_filetype("file.xml") == "bytes"
def test_verify_or_create_sops_branch(self):
"""Verify or create the sops branch"""
@@ -171,6 +164,17 @@ class TreeTest(unittest2.TestCase):
cleartree = sops.walk_and_decrypt(OrderedDict(crypttree), key, isRoot=True)
assert cleartree == tree
def test_bytes_encrypt_and_decrypt(self):
"""Test encryption/decryption of numbers"""
key = os.urandom(32)
tree = OrderedDict()
tree['data'] = os.urandom(4096)
tree['sops'] = dict()
crypttree = sops.walk_and_encrypt(OrderedDict(tree), key, isRoot=True)
assert tree['sops']['mac'].startswith("ENC[AES256_GCM,data:")
cleartree = sops.walk_and_decrypt(OrderedDict(crypttree), key, isRoot=True)
assert cleartree == tree
def test_walk_list_and_encrypt(self):
"""Walk a list contained in a branch and encrypts its values."""
# - test stash value
@@ -250,10 +254,10 @@ class TreeTest(unittest2.TestCase):
with mock.patch.object(builtins, 'open', m):
assert sops.validate_syntax('path', 'yaml') == True
def test_text_syntax(self):
def test_bytes_syntax(self):
m = mock.mock_open(read_data=sops.DEFAULT_TEXT)
with mock.patch.object(builtins, 'open', m):
assert sops.validate_syntax('path', 'text') == True
assert sops.validate_syntax('path', 'bytes') == True
def test_subtree(self):
"""Extract a subtree from a document."""