From 51e13b1589a59a53545637e735a371df3f0848ea Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Tue, 19 Apr 2016 12:22:32 -0400 Subject: [PATCH] Fix handling of binary data to preserve integrity, fixes #59 --- Makefile | 32 ++++++++++++++++---------------- example.json | 14 +------------- example.txt | 14 +------------- example.yaml | 7 ------- setup.py | 2 +- sops/__init__.py | 28 ++++++++++++++-------------- 6 files changed, 33 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 5aceecf09..6ba875509 100644 --- a/Makefile +++ b/Makefile @@ -59,29 +59,29 @@ functional-tests: python3.4 sops/__init__.py -d /tmp/testdata2.7.$$type > /dev/null || exit 1; \ done && \ for ver in 2.6 2.7 3.4; do \ - echo "Testing Python$$ver round-trip on binary file" && \ - dd if=/dev/urandom of=/tmp/testdata-$$ver-randomfile bs=1024 count=1024 2>&1 1>/dev/null && \ - python$$ver sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata-$$ver-randomfile > /tmp/testdata-$$ver-randomfile.enc && \ - python$$ver sops/__init__.py -d /tmp/testdata-$$ver-randomfile.enc > /tmp/testdata-$$ver-randomfile.dec && \ - if [ $$(sha256sum /tmp/testdata-$$ver-randomfile | cut -d ' ' -f 1) != $$(sha256sum /tmp/testdata-$$ver-randomfile.dec | cut -d ' ' -f 1) ]; then \ - echo "Binary file roundtrip failed, checksum doesn't match"; exit 0; \ - else \ - echo "Binary file roundtrip succeeded"; \ - fi; \ done functional-tests-once: gpg --import tests/sops_functional_tests_key.asc 2>&1 1>/dev/null || exit 0 for type in yaml json txt; do \ - echo "Testing $$type decryption" && \ - python sops/__init__.py -d example.$$type > /tmp/testdata.$$type && \ - echo "Testing $$type encryption" && \ + echo "Testing $$type decryption"; \ + python sops/__init__.py -d example.$$type > /tmp/testdata.$$type; \ + echo "Testing $$type encryption" ; \ python sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata.$$type > /tmp/testdataenc.$$type; \ - echo "Testing $$type re-decryption" && \ - python sops/__init__.py -d /tmp/testdataenc.$$type > /dev/null || exit 1; \ - echo "Testing removing PGP key to $$type encrypted file" && \ - python sops/__init__.py -r --rm-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 /tmp/testdataenc.$$type || exit 1; \ + echo "Testing $$type re-decryption" ; \ + python sops/__init__.py -d /tmp/testdataenc.$$type > /dev/null ; \ + echo "Testing removing PGP key to $$type encrypted file" ; \ + python sops/__init__.py -r --rm-pgp 85D77543B3D624B63CEA9E6DBC17301B491B3F21 /tmp/testdataenc.$$type ; \ done + echo "Testing round-trip on binary file" + dd if=/dev/urandom of=/tmp/testdata-randomfile bs=1024 count=1024 2>&1 1>/dev/null + python sops/__init__.py -e -p "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A" /tmp/testdata-randomfile > /tmp/testdata-randomfile.enc + python sops/__init__.py -d /tmp/testdata-randomfile.enc > /tmp/testdata-randomfile.dec + if [ $$(sha256sum /tmp/testdata-randomfile | cut -d ' ' -f 1) != $$(sha256sum /tmp/testdata-randomfile.dec | cut -d ' ' -f 1) ]; then \ + echo "Binary file roundtrip failed, checksum doesn't match"; exit 0; \ + else \ + echo "Binary file roundtrip succeeded"; \ + fi; pypi: $(PYTHON) setup.py sdist check upload --sign diff --git a/example.json b/example.json index d968d176d..98ea2ab26 100644 --- a/example.json +++ b/example.json @@ -22,18 +22,6 @@ "sops": { "version": 1.6, "mac": "ENC[AES256_GCM,data:YlhAZo7NAUHlRQzAPoPha12yl3nEaaq3lKyJ1hoMMtw9M7kts1cfrk301arKHrRAssqcqq1RizYVTyennOIj+TpOYoi8dOzXAtm+NrjQrb7SNosGLPXO2mbvOIoeqzZuzureHRCitNeLwzD8+U/vlGhRTuB6Be7TGt55CD+OMMM=,iv:x9DFpKn9xlq5uoRfbV2rnAsmjY4YyZH6wQnBY80x2dw=,tag:JnGt3eJNLRC3FamfNkAkew==,type:str]", - "kms": [ - { - "created_at": "2015-11-25T14:34:39Z", - "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzZr3EKR6/6z7Bg0MECARCAOyKbAecKpjU5xADsXg3XLgqs10pr7t9CjpSgYYw/oq3IkSMhED+jZ5RzpRByMSOcl7XOPVShTBP0UROI", - "arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e" - }, - { - "created_at": "2015-11-25T14:34:39Z", - "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxrelu7r4H0CTrjEl4CARCAOxg41AwzpQPlfAKPAg9EJCF2jl1j61m1hAW3GstrT90j9xTVHsG21xrJHcQOPK/+X/AEr5fjaMLPMEnq", - "arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d" - } - ], "pgp": [ { "fp": "1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A", @@ -49,4 +37,4 @@ "lastmodified": "2016-02-03T18:46:24Z", "attention": "This section contains key material that should only be modified with extra care. See `sops -h`." } -} \ No newline at end of file +} diff --git a/example.txt b/example.txt index 58042c661..3ee315940 100644 --- a/example.txt +++ b/example.txt @@ -3,18 +3,6 @@ "sops": { "mac": "ENC[AES256_GCM,data:OQnRHfLfaxsMclFLTsNoog7JvEyMOBcXWQ/qGRmAPQmHW4pM2nzW1n8gdVcSGTA0AHz1Dgi3HW1r9E23z2Kf6K+D1JXSed+krs9BgrMO4Cc0mbPme+dYrCcOOYuKWmwlW6n9X+OiTzJ8tQvOJHGBGjs1+829F+Y2DZ1n5/62cHQ=,iv:gx99cyKW1IfA1q1rKvSRLBauntaAxwIta3Tqq6b8NL4=,tag:eTAYmXIkgHIdn/JKTIqm0w==,type:str]", "version": 1.0, - "kms": [ - { - "arn": "arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e", - "created_at": "2015-11-25T15:05:50Z", - "enc": "CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxJepR1L44IuTqAaAcCARCAO6Z7vbt7t9Qk32TbLhES3u1HLmDgUZm3P3Hnl8lRN0SKJanQGssKcvNA5v5nqoXUDc/M5ylscImL3SoS" - }, - { - "arn": "arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d", - "created_at": "2015-11-25T15:05:50Z", - "enc": "CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAy/yi0dBkO1p+AE3D0CARCAO7tM7TaoI4OBA0FzIjdo4NEdOWi1oKBKf28jji0z9tvvUZEcaG8PcKH4u73nRMQBL5uo+zjagXfeQJhL" - } - ], "pgp": [ { "created_at": "2015-11-25T15:05:50Z", @@ -30,4 +18,4 @@ "lastmodified": "2015-11-25T15:05:50Z", "attention": "This section contains key material that should only be modified with extra care. See `sops -h`." } -} \ No newline at end of file +} diff --git a/example.yaml b/example.yaml index f11aaa078..35a477e8a 100644 --- a/example.yaml +++ b/example.yaml @@ -34,13 +34,6 @@ nested_unencrypted: is: all: going to remain in clear text sops: - kms: - - created_at: '2015-11-25T00:32:57Z' - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyzrMwHaX8rsBh/iNACARCAO/eeScqy8gZpfvDoHilBD+cw+1n6iFsTQmEQJro4QY8p+LUXSLFsnUge8xcADZrIGBup9BBJbdR+qyot - arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e - - created_at: '2015-11-25T00:32:57Z' - enc: CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzonxxlGDduanr16MwCARCAO70FBqnx7K2xaY8++gATYtsLgJfq5aW8lRWK515g5fEDpn/+PbrGSY9YxsFul024+fIev+8r3AKDX7K3 - arn: arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d pgp: - fp: 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A created_at: '2015-11-25T00:32:57Z' diff --git a/setup.py b/setup.py index ae2e12b8a..07f42bf4a 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ with codecs.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: setup( name="sops", py_modules=['sops'], - version="1.11", + version="1.12", author="Julien Vehent", author_email="jvehent@mozilla.com", description="Secrets OPerationS (sops) is an editor of encrypted files", diff --git a/sops/__init__.py b/sops/__init__.py index 026a3dfc1..7885c8e20 100644 --- a/sops/__init__.py +++ b/sops/__init__.py @@ -1243,27 +1243,27 @@ def write_file(tree, path=None, filetype=None): else: fd.write(jsonstr.encode('utf-8')) else: + # BINARY format if 'data' in tree: - try: - if path == 'stdout': - sys.stdout.write(tree['data']) - else: - fd.write(tree['data'].encode('utf-8')) - except: - if path == 'stdout': - sys.stdout.write(tree['data'].decode('utf-8')) - else: - fd.write(tree['data']) + # binary data is stored in json format under a key called "data". + # we simply write the content of this key as is to the output file if path == 'stdout': - sys.stdout.write("\n") + if (sys.version_info[0] == 3 and + isinstance(tree['data'], bytes)): + sys.stdout.buffer.write(tree['data']) + else: + sys.stdout.write(tree['data']) else: - fd.write("\n") + try: + fd.write(tree['data'].encode('utf-8')) + except: + fd.write(tree['data']) if 'sops' in tree: jsonstr = json.dumps(tree['sops'], sort_keys=True) if path == 'stdout': - sys.stdout.write("SOPS=%s" % jsonstr) + sys.stdout.write("\nSOPS=%s" % jsonstr) else: - fd.write("SOPS=%s" % jsonstr.encode('utf8')) + fd.write("\nSOPS=%s" % jsonstr.encode('utf8')) if path != 'stdout': fd.close() return path