diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..268f73b19 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +all: + ./setup.py build + +install: + ./setup.py install + +rpm: + fpm -s python -t rpm -d pytz -d python-requests-futures ./setup.py + +deb: + fpm -s python -t deb ./setup.py + +tests: test +test: + python ./sops -d example.yaml + +pypi: + python setup.py sdist check upload --sign + +clean: + rm -rf *pyc + rm -rf build + rm -rf __pycache__ + rm -rf dist diff --git a/README.rst b/README.rst index 6fb12c04f..efbf588fb 100644 --- a/README.rst +++ b/README.rst @@ -1,15 +1,127 @@ SOPS: Secrets OPerationS ======================== -`sops` is a cli that encrypt values of yaml, json or text files using AWS KMS. +`sops` is a secrets management tool that encrypts YAML, JSON and TEXT files +using AWS KMS and/or PGP (via GnuPG). + +Requirements +------------ +First install some libraries from your package manager: + +* RHEL family:: + + sudo yum install libyaml-devel python-devel libffi-devel + +* Debian family:: + + sudo apt-get install libyaml-dev python-dev libffi-dev + +Then install the python requirements from pip:: + + pip install boto3 ruamel.yaml cryptography + git clone https://github.com/mozilla-services/sops.git + cd sops && ./sops -h + +* `boto3 `_ +* `ruamel.yaml `_ +* `cryptography `_ Usage ----- -Editing -~~~~~~~ +If you're using AWS KMS, create one or multiple master keys in the IAM console +and export them, comma separated, in the **SOPS_KMS_ARN** env variable. It is +recommended to use at least two master keys in different regions. -`sops` encrypted file contain the necessary KMS information to decrypt their -content. All a user of `sops` need is valid AWS credentials and the necessary +.. code:: bash + + export SOPS_KMS_ARN="arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e,arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d" + +Your AWS credentials must be present in `~/.aws/credentials`. sops uses boto3. + +.. code:: + + $ cat ~/.aws/credentials + [default] + aws_access_key_id = AKI..... + aws_secret_access_key = mw...... + +If you want to use PGP, export the fingerprints of the public keys, comma +separated, in the **SOPS_PGP_FP** env variable. + +.. code:: bash + + export SOPS_PGP_FP="85D77543B3D624B63CEA9E6DBC17301B491B3F21,E60892BB9BD89A69F759A1A0A3D652173B763E8F" + +Note: you can use both PGP and KMS simultaneously. + +Then simply call `sops` with a file path as argument. It will handle the +encryption/decryption transparently and open the cleartext file in an editor. + +.. code:: bash + + $ sops mynewtestfile.yaml + mynewtestfile.yaml doesn't exist, creating it. + please wait while an encryption key is being generated and stored in a secure fashion + [... editing happens in vim, or whatever $EDITOR is set to ...] + file written to mynewtestfile.yaml + +.. code:: yaml + + myapp1: ENC[AES256_GCM,data:Tr7oc19nc6t1m9OrUeo=,iv:1vzlPZLfy6wa14/x17P8Ix8wEGDeY0v2dIboZmmwpww=,aad:NpobRzMzpDOkqijzONm8KglltzG+aBV7BJAxtm77veo=,tag:kaYqRgGGBhXhODSSmIZwyA==] + app2: + db: + user: ENC[AES256_GCM,data:CwE4O1s=,iv:S0fozGAOxNma/pWDUuk1iEaYw0wlba0VOLHjPxIok2k=,aad:nEVizsMMyBXOxySnOHw/trTFBSW72nh+Q80YU7TPgIo=,tag:XaGsYaL9LCkLWJI0uxnTYw==] + password: ENC[AES256_GCM,data:p673JCgHYw==,iv:EOOeivCp/Fd80xFdMYX0QeZn6orGTK8CeckmipjKqYY=,aad:UAhi/SHK0aCzptnFkFG4dW8Vv1ASg7TDHD6lui9mmKQ=,tag:QE6uuhRx+cGInwSVdmxXzA==] + # private key for secret operations in app2 + key: |- + ENC[AES256_GCM,data:Ea3zTFSOlg1PDZmBa1U2dtKl3pO4nTmaFswJx41fPfq3u8O2/Bq1UVfXn2SrO13obfr6xH4zuUceCDTvW2qvphlan5ir609EXt4dE2TEEcjVKhmAHf4LMwlZVAbvTJtlsnvo/aYJH95uctjsSX5h8pBlLaTGBGYwMrZuMyRU6vdcMWyha+piJckUc9sq7fevy1TSqIxf1Usbn/0NEklWm2VSNzQ2Urqtny6EXar+xU7NfYSRJ3mqmcJZ14oIeXPdpk962RwMEFWdYrbE7D59kWU2BgMjDxYJD5KXpWiw2YCrA/wsATxVCbZlwqC+TJFA5WAUZX756mFhV/t2Li3zQyDNUe6KkMXV9qwf/oV1j5sVRVFsKDYIBqhi3qWBVA+SO9RloQMjhru+IsdbQcS4LKq/1DrBENeZuJ0djUAxKLVfJzMGUf89ju3m9IEPovW8mfF0RbfAGRwFHMO9nEXCxrTLERf3owdR3u4j5/rNBpIvvy1z+2dy6sAx/eyNdS+cn5qO9BPAxsXpSwkaI96rlBagwH1Pfxus0x/D00j93OpE+M8MgQ/9LA68FlCFU4OAQlvw8f7MPoxnq+/+gFTS/qqjTR6EoUuX5NH2WY93YCC5TCbe4GOXyP0H05PbIWq55UMVLNcpAyac3gO4kL5O5U8=,iv:Dl61tsemKH0fdmNul/PmEEsRYFAh8GorR8GRupus/EM=,aad:Ft2aSYYukD1x8pMj1WvmodLjJV6waPy5FqdlImWyQKA=,tag:EPg4KpWqni/buCFjFL857A==] + an_array: + - ENC[AES256_GCM,data:v8dfh92oL8IcgjQ=,iv:HgNNPlQh9GNdE+YPvG4Ufpb2I0sIlEpCsOW3lJA1uBE=,aad:21GroP5gb9sCTxZIahN1NhMGqRPQZZksAr5Q7eCeHRc=,tag:gLsjVqot9+Pqck9LJC+bVA==] + - ENC[AES256_GCM,data:X1LMy27AE9SI4h0=,iv:oA1kSg9esGxAvi3qhpcM6Ewrh+p0CFV5cgf6jSPpM08=,aad:CZ7FGJNko6367sd6PwbrIgN/V7Rly4TptbQ1gVsXT1Q=,tag:HerE4nTstX2QZhMn3CPZcw==] + - ENC[AES256_GCM,data:KNkH9iI0bSyvcP3E+BRbqfcPUv3YBbCmtvbK1y+sHMI6Z1kXnkX4RoyYiZZXrM680Nh/p0TxNOdNsA==,iv:1h3KbThwTsRaVF+k+dnSwfocSEoyT00X279Dg1Wro60=,aad:foCwpM862VeAD2/7bHRJHAYISneTUJweoSRl2oAdsI4=,tag:tNuCjsNqIy5FVDRu39dQcw==] + sops: + kms: + - created_at: 1441570389.775376 + enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAgB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxn6jfG4e44/phCddICARCAOzfGN/7WlU0MouQRXv22Pix46dSocMH1K7Xf47WqF1rCEcuN1aMVBj+IxwOgOVxVsr0Kze4lnMqPm1Hm + arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e + - created_at: 1441570391.925734 + enc: CiBdfsKZbRNf/Li8Tf2SjeSdP76DineB1sbPjV0TV+meTxKnAQEBAgB4XX7CmW0TX/y4vE39ko3knT++g4p3gdbGz41dE1fpnk8AAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxGzsadorzSGbp73+ECARCAO0hc3cYxgNF2OU5TfTj8iyt/S6DTKDO+gwcHc3sy3ELQ/pUjSFJScYOQmqYpvsznhZ4YjHQWDdbRawNx + arn: arn:aws:kms:ap-southeast-1:656532927350:key/9006a8aa-0fa6-4c14-930e-a2dfb916de1d + pgp: + - fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21 + created_at: 1441570391.930042 + enc: | + -----BEGIN PGP MESSAGE----- + Version: GnuPG v1 + + hQIMA0t4uZHfl9qgAQ//UvGAwGePyHuf2/zayWcloGaDs0MzI+zw6CmXvMRNPUsA + pAgRKczJmDu4+XzN+cxX5Iq9xEWIbny9B5rOjwTXT3qcUYZ4Gkzbq4MWkjuPp/Iv + qO4MJaYzoH5YxC4YORQ2LvzhA2YGsCzYnljmatGEUNg01yJ6r5mwFwDxl4Nc80Cn + RwnHuGExK8j1jYJZu/juK1qRbuBOAuruIPPWVdFB845PA7waacG1IdUW3ZtBkOy3 + O0BIfG2ekRg0Nik6sTOhDUA+l2bewCcECI8FYCEjwHm9Sg5cxmP2V5m1mby+uKAm + kewaoOyjbmV1Mh3iI1b/AQMr+/6ZE9MT2KnsoWosYamFyjxV5r1ZZM7cWKnOT+tu + KOvGhTV1TeOfVpajNTNwtV/Oyh3mMLQ0F0HgCTqomQVqw5+sj7OWAASuD3CU/dyo + pcmY5Qe0TNL1JsMNEH8LJDqSh+E0hsUxdY1ouVsg3ysf6mdM8ciWb3WRGxih1Vmf + unfLy8Ly3V7ZIC8EHV8aLJqh32jIZV4i2zXIoO4ZBKrudKcECY1C2+zb/TziVAL8 + qyPe47q8gi1rIyEv5uirLZjgpP+JkDUgoMnzlX334FZ9pWtQMYW4Y67urAI4xUq6 + /q1zBAeHoeeeQK+YKDB7Ak/Y22YsiqQbNp2n4CKSKAE4erZLWVtDvSp+49SWmS/S + XgGi+13MaXIp0ecPKyNTBjF+NOw/I3muyKr8EbDHrd2XgIT06QXqjYLsCb1TZ0zm + xgXsOTY3b+ONQ2zjhcovanDp7/k77B+gFitLYKg4BLZsl7gJB12T8MQnpfSmRT4= + =oJgS + -----END PGP MESSAGE----- + +A copy of the encryption/decryption key is stored securely in each KMS and PGP +block. As long as one of the KMS or PGP method is still usable, you will be able +to access you data. + +To decrypt a file in a `cat` fashion, use the `-d` flag: + +.. code:: bash + + $ sops -d mynewtestfile.yaml + +`sops` encrypted files contain the necessary information to decrypt their content. +All a user of `sops` need is valid AWS credentials and the necessary permissions on KMS keys. Given that, the only command a `sops` user need is: @@ -19,109 +131,92 @@ Given that, the only command a `sops` user need is: $ sops `` will be opened, decrypted, passed to a text editor (vim by default), -encrypted if modified, and save back to its original location. All of these +encrypted if modified, and saved back to its original location. All of these steps, apart from the actual editing, are transparent to the user. -Creating -~~~~~~~~ +Cryptographic details +--------------------- -In order to create a file, the KMS ARN must be provided to `sops`, either on the -command line in the `-k` flag, or in the environment variable **SOPS_KMS_ARN**. +When sops creates a file, it generates a random 256 bits data key and asks each +KMS and PGP master key to encrypt the data key. The encrypted version of the data +key is stored in the `sops` metadata under `sops.kms` and `sops.pgp`. -`sops` automatically create a file if the given path doesn't exist (it will not -create folders, however). - -.. code:: bash - - $ sops newfile.yaml -k arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e - newfile.yaml doesn't exist, creating it. - new data key generated from kms: CiC6yCOtzsnFhkfdIs... - file written to newfile.yaml - -Input some cleartext yaml: +For KMS: .. code:: yaml - myapp1: t00m4nys3cr3tz - app2: - db: - user: bob - password: c4r1b0u - # private key for secret operations in app2 - key: | - -----BEGIN RSA PRIVATE KEY----- - MIIBPAIBAAJBAPTMNIyHuZtpLYc7VsHQtwOkWYobkUblmHWRmbXzlAX6K8tMf3Wf - Erb0xAEyVV7e8J0CIQC8VBY8f8yg+Y7Kxbw4zDYGyb3KkXL10YorpeuZR4LuQQIg - bKGPkMM4w5blyE1tqGN0T7sJwEx+EUOgacRNqM2ljVA= - -----END RSA PRIVATE KEY----- - an_array: - - secretuser1 # a super secret user - - secretuser2 sops: kms: - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usg...... - enc_ts: 1439587921.752637 + - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyGdRODuYMHbA8Ozj8CARCAO7opMolPJUmBXd39Zlp0L2H9fzMKidHm1vvaF6nNFq0ClRY7FlIZmTm4JfnOebPseffiXFn9tG8cq7oi + enc_ts: 1439568549.245995 arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e -After saving the file and exiting, it is automatically encrypted. Keys are -still in cleartext, but value are now unreadable. +For PGP: .. code:: yaml - myapp1: ENC[AES256_GCM,data=s4mlbkPqyk+GFDluAHY=,iv=7c9X8CwZyK5PsRRmUpzxL4CeQmp7+ry6mVemJtmpR7U=,aad=CFVNHUiz8xupOCMNYUlF4l+TcCjGaxayiknL9tQtolw=,tag=5ecBRedoXPJJ3uBjaj7J1w==] - app2: - db: - user: ENC[AES256_GCM,data=CmkT,iv=xnUTxXU4g5lKEqetiZrM2s+m20idUUt9xGU6XitsIic=,aad=KidFJD6ioPXKqz+BYVYXtHk8Dd6e1yvhPx6kO5BOJTs=,tag=7WDZbBf2oqMuXi3YH4m2Ig==] - password: ENC[AES256_GCM,data=zw2yh6Oz8Q==,iv=Apme9l8h+OwdwgbozsuXa1mVK+b821eoQNEBBSF6Ihs=,aad=SZFoaQDlNe0SkRaX65zB7E8SDyhkr9uVBI+3GWUBKsQ=,tag=We5dwW455S1M4ob1HzAu7Q==] - # private key for secret operations in app2 - key: |- - ENC[AES256_GCM,data=feo0o1qW8p4Nw2tN9/QAt0zoeGZHgolORWXH+7hk4Oc5nQcA/Ve3mYQ9TKSZAtzsYr+OEnEVUAg/RzvXy40F9dXsv2ugux+DLS1SIWddKRAdeL283vjnsDtydc3+AP+UuEuCyIVHqT8uKcqQnenzzu/yx07scIwcMQ6Vs2RnQ3WwrOrkBsbPQ4PpuPsrlck7EbcKkMnoIe09AMN3/J3A+mlmOGBxAio1ahFXpeBzzYeoRkjffojvigT2ULZy92Kx1afRSnWXNmUtMKqbJDIIvYulWHW5efAnulk/nHZ0Bhy+wxV0jqXAp7mKiIlGuydxRZ6DPon7jhABWV5d93EZdZJ+/33sUiOyQKIEukqae17C2Hrt0QoGg7OhG/O0oyTKiql0Nj6KC3bFbkdM7sSFbsIbv/of0P5Kb2zr3VYAJriJqUWMKzj3i7M6z9+wxrTVxuMQ4Lvzw3aHDjNOgJobkfjxxMYUvF5l2OWRFrdxtY9WxBYAcDzkJnBYtkPnUzlEc/8ieypefqOBlcphOvzl+EjM1I0N4OGG5ij5nNsHQ/MSoM3FJpjROQKclhz8ZN5CH41LUemP3AdddPpoUuwzHCxR8NskUhyHBlep0iZL9xGFLL7SwYEACKxk2BCwHMWeNmXfKo6co+wjCmn+un3FANE=,iv=NworRcR7VnLgW30c4W9OmVgBaY7tA1fd090JQpBM5ho=,aad=sbwFbTuEr9FbPd/ofR7BL9NORUpfmNd+X3Q+tJqmj8g=,tag=wc7RWWBArQrTMt3AAbSwZQ==] - an_array: - - ENC[AES256_GCM,data=L3Y0Bzn2M6yERcU=,iv=FslXY0z783MXhjCaz9ZZTqNaEwBWZkspNHAtHJaENH0=,aad=x0x9+PnDW81oLbYufq72RmaRZB29IPCALCL94KtmsvQ=,tag=qPyqJ3I9JM6wIJDOmgmJkQ==] - - ENC[AES256_GCM,data=To5dwUDJi4Mh3hc=,iv=03vcf/AJaUKcHKEnGPq7ih8/xaKHewYiFkQcWOsh7So=,aad=nxUVG7rA+TjyK9BrzVtDGbCp7Iu7BCRLjYvZSnI5iCI=,tag=41ExX9KH+jRYvn51aaP6OA==] sops: - kms: - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwkRAZG5vQyIKvIKPwCARCAO9zQ43qeQ8loKu0HzXRnpqi6MK/+TpbO22sH0NkVXddXNTl7lfPjKc6gJynrEVdu6aCslUYIid+3FONY - enc_ts: 1439587921.752637 - arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e + pgp: + - fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21 + created_at: 1441570391.930042 + enc: | + -----BEGIN PGP MESSAGE----- + Version: GnuPG v1 -To decrypt, using flag `-d`. + hQIMA0t4uZHfl9qgAQ//UvGAwGePyHuf2/zayWcloGaDs0MzI+zw6CmXvMRNPUsA + pAgRKczJmDu4+XzN+cxX5Iq9xEWIbny9B5rOjwTXT3qcUYZ4Gkzbq4MWkjuPp/Iv + qO4MJaYzoH5YxC4YORQ2LvzhA2YGsCzYnljmatGEUNg01yJ6r5mwFwDxl4Nc80Cn + RwnHuGExK8j1jYJZu/juK1qRbuBOAuruIPPWVdFB845PA7waacG1IdUW3ZtBkOy3 + O0BIfG2ekRg0Nik6sTOhDUA+l2bewCcECI8FYCEjwHm9Sg5cxmP2V5m1mby+uKAm + kewaoOyjbmV1Mh3iI1b/AQMr+/6ZE9MT2KnsoWosYamFyjxV5r1ZZM7cWKnOT+tu + KOvGhTV1TeOfVpajNTNwtV/Oyh3mMLQ0F0HgCTqomQVqw5+sj7OWAASuD3CU/dyo + pcmY5Qe0TNL1JsMNEH8LJDqSh+E0hsUxdY1ouVsg3ysf6mdM8ciWb3WRGxih1Vmf + unfLy8Ly3V7ZIC8EHV8aLJqh32jIZV4i2zXIoO4ZBKrudKcECY1C2+zb/TziVAL8 + qyPe47q8gi1rIyEv5uirLZjgpP+JkDUgoMnzlX334FZ9pWtQMYW4Y67urAI4xUq6 + /q1zBAeHoeeeQK+YKDB7Ak/Y22YsiqQbNp2n4CKSKAE4erZLWVtDvSp+49SWmS/S + XgGi+13MaXIp0ecPKyNTBjF+NOw/I3muyKr8EbDHrd2XgIT06QXqjYLsCb1TZ0zm + xgXsOTY3b+ONQ2zjhcovanDp7/k77B+gFitLYKg4BLZsl7gJB12T8MQnpfSmRT4= + =oJgS + -----END PGP MESSAGE----- -.. code:: bash +sops then opens a text editor on the newly created file. The user adds data to the +file and saves it when done. - $ sops -d newfile.yaml - myapp1: t00m4nys3cr3tz - app2: - db: - user: bob - [...] +Upon save, sops browses the entire file as of a key/value tree. Every time sops +encounters a leaf value (a value that does not have children), it encrypts the +value with AES256_GCM using the data key, a 256 bits random initialization vector +and 256 bits of random additional data. While the same data key is used to +encrypt all values of a document, each value receives a unique initialization +vector and unique authentication data. -Set the env variable **SOPS_KMS_ARN** to your KMS ARN value to avoid -needing to set the `-k` flag every time you create a file. +The result of AES256_GCM encryption is stored in the leaf of the tree using a +simple key/value format:: -.. code:: bash + ENC[AES256_GCM, + data:CwE4O1s=, + iv:S0fozGAOxNma/pWDUuk1iEaYw0wlba0VOLHjPxIok2k=, + aad:nEVizsMMyBXOxySnOHw/trTFBSW72nh+Q80YU7TPgIo=, + tag:XaGsYaL9LCkLWJI0uxnTYw==] - $ export SOPS_KMS_ARN="arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e" - $ sops newfile.yaml +where: -Requirements ------------- -* `boto3 `_ -* `ruamel.yaml `_; requires - libyaml-devel and python-devel prior to `pip install`-ing it. +* **data** is the encrypted value +* **iv** is the 256 bits initialization vector +* **aad** is the 256 bits additional data +* **tag** is the authentication tag -.. code:: +The encrypted file is written to disk with nested keys in cleartext and +encrypted values. We expect that keys do not carry sensitive information, and +keeping them in cleartext allows for better diff and overall readability. - sudo yum install libyaml-devel python-devel - sudo pip install ruamel.yaml +Any valid KMS or PGP master key can later decrypt the data key and access the +data. -* `cryptography `_; requires - libffi-devel prior to `pip install`-ing it. - -.. code:: - - sudo yum install libffi-devel - sudo pip install cryptography +Multiple master keys allow for sharing encrypted files without sharing master +keys, and provide disaster recovery solution. The recommended way to use sops +is to have two KMS master keys in different region and one PGP public key with +the private key stored offline. If, by any chance, both KMS master keys are +lost, you can always recover the encrypted data using the PGP private key. License ------- @@ -129,7 +224,7 @@ Mozilla Public License Version 2.0 Authors ------- -* Julien Vehent +* Julien Vehent Credits ------- diff --git a/example.yaml b/example.yaml index ace481e2c..20b0c9846 100644 --- a/example.yaml +++ b/example.yaml @@ -1,17 +1,17 @@ -myapp1: ENC[AES256_GCM,data=Tr7oc19nc6t1m9OrUeo=,iv=1vzlPZLfy6wa14/x17P8Ix8wEGDeY0v2dIboZmmwpww=,aad=NpobRzMzpDOkqijzONm8KglltzG+aBV7BJAxtm77veo=,tag=kaYqRgGGBhXhODSSmIZwyA==] +myapp1: ENC[AES256_GCM,data:Tr7oc19nc6t1m9OrUeo=,iv:1vzlPZLfy6wa14/x17P8Ix8wEGDeY0v2dIboZmmwpww=,aad:NpobRzMzpDOkqijzONm8KglltzG+aBV7BJAxtm77veo=,tag:kaYqRgGGBhXhODSSmIZwyA==] app2: db: - user: ENC[AES256_GCM,data=CwE4O1s=,iv=S0fozGAOxNma/pWDUuk1iEaYw0wlba0VOLHjPxIok2k=,aad=nEVizsMMyBXOxySnOHw/trTFBSW72nh+Q80YU7TPgIo=,tag=XaGsYaL9LCkLWJI0uxnTYw==] - password: ENC[AES256_GCM,data=p673JCgHYw==,iv=EOOeivCp/Fd80xFdMYX0QeZn6orGTK8CeckmipjKqYY=,aad=UAhi/SHK0aCzptnFkFG4dW8Vv1ASg7TDHD6lui9mmKQ=,tag=QE6uuhRx+cGInwSVdmxXzA==] + user: ENC[AES256_GCM,data:CwE4O1s=,iv:S0fozGAOxNma/pWDUuk1iEaYw0wlba0VOLHjPxIok2k=,aad:nEVizsMMyBXOxySnOHw/trTFBSW72nh+Q80YU7TPgIo=,tag:XaGsYaL9LCkLWJI0uxnTYw==] + password: ENC[AES256_GCM,data:p673JCgHYw==,iv:EOOeivCp/Fd80xFdMYX0QeZn6orGTK8CeckmipjKqYY=,aad:UAhi/SHK0aCzptnFkFG4dW8Vv1ASg7TDHD6lui9mmKQ=,tag:QE6uuhRx+cGInwSVdmxXzA==] # private key for secret operations in app2 key: |- - ENC[AES256_GCM,data=Ea3zTFSOlg1PDZmBa1U2dtKl3pO4nTmaFswJx41fPfq3u8O2/Bq1UVfXn2SrO13obfr6xH4zuUceCDTvW2qvphlan5ir609EXt4dE2TEEcjVKhmAHf4LMwlZVAbvTJtlsnvo/aYJH95uctjsSX5h8pBlLaTGBGYwMrZuMyRU6vdcMWyha+piJckUc9sq7fevy1TSqIxf1Usbn/0NEklWm2VSNzQ2Urqtny6EXar+xU7NfYSRJ3mqmcJZ14oIeXPdpk962RwMEFWdYrbE7D59kWU2BgMjDxYJD5KXpWiw2YCrA/wsATxVCbZlwqC+TJFA5WAUZX756mFhV/t2Li3zQyDNUe6KkMXV9qwf/oV1j5sVRVFsKDYIBqhi3qWBVA+SO9RloQMjhru+IsdbQcS4LKq/1DrBENeZuJ0djUAxKLVfJzMGUf89ju3m9IEPovW8mfF0RbfAGRwFHMO9nEXCxrTLERf3owdR3u4j5/rNBpIvvy1z+2dy6sAx/eyNdS+cn5qO9BPAxsXpSwkaI96rlBagwH1Pfxus0x/D00j93OpE+M8MgQ/9LA68FlCFU4OAQlvw8f7MPoxnq+/+gFTS/qqjTR6EoUuX5NH2WY93YCC5TCbe4GOXyP0H05PbIWq55UMVLNcpAyac3gO4kL5O5U8=,iv=Dl61tsemKH0fdmNul/PmEEsRYFAh8GorR8GRupus/EM=,aad=Ft2aSYYukD1x8pMj1WvmodLjJV6waPy5FqdlImWyQKA=,tag=EPg4KpWqni/buCFjFL857A==] + ENC[AES256_GCM,data:Ea3zTFSOlg1PDZmBa1U2dtKl3pO4nTmaFswJx41fPfq3u8O2/Bq1UVfXn2SrO13obfr6xH4zuUceCDTvW2qvphlan5ir609EXt4dE2TEEcjVKhmAHf4LMwlZVAbvTJtlsnvo/aYJH95uctjsSX5h8pBlLaTGBGYwMrZuMyRU6vdcMWyha+piJckUc9sq7fevy1TSqIxf1Usbn/0NEklWm2VSNzQ2Urqtny6EXar+xU7NfYSRJ3mqmcJZ14oIeXPdpk962RwMEFWdYrbE7D59kWU2BgMjDxYJD5KXpWiw2YCrA/wsATxVCbZlwqC+TJFA5WAUZX756mFhV/t2Li3zQyDNUe6KkMXV9qwf/oV1j5sVRVFsKDYIBqhi3qWBVA+SO9RloQMjhru+IsdbQcS4LKq/1DrBENeZuJ0djUAxKLVfJzMGUf89ju3m9IEPovW8mfF0RbfAGRwFHMO9nEXCxrTLERf3owdR3u4j5/rNBpIvvy1z+2dy6sAx/eyNdS+cn5qO9BPAxsXpSwkaI96rlBagwH1Pfxus0x/D00j93OpE+M8MgQ/9LA68FlCFU4OAQlvw8f7MPoxnq+/+gFTS/qqjTR6EoUuX5NH2WY93YCC5TCbe4GOXyP0H05PbIWq55UMVLNcpAyac3gO4kL5O5U8=,iv:Dl61tsemKH0fdmNul/PmEEsRYFAh8GorR8GRupus/EM=,aad:Ft2aSYYukD1x8pMj1WvmodLjJV6waPy5FqdlImWyQKA=,tag:EPg4KpWqni/buCFjFL857A==] an_array: -- ENC[AES256_GCM,data=v8dfh92oL8IcgjQ=,iv=HgNNPlQh9GNdE+YPvG4Ufpb2I0sIlEpCsOW3lJA1uBE=,aad=21GroP5gb9sCTxZIahN1NhMGqRPQZZksAr5Q7eCeHRc=,tag=gLsjVqot9+Pqck9LJC+bVA==] -- ENC[AES256_GCM,data=X1LMy27AE9SI4h0=,iv=oA1kSg9esGxAvi3qhpcM6Ewrh+p0CFV5cgf6jSPpM08=,aad=CZ7FGJNko6367sd6PwbrIgN/V7Rly4TptbQ1gVsXT1Q=,tag=HerE4nTstX2QZhMn3CPZcw==] -- ENC[AES256_GCM,data=KNkH9iI0bSyvcP3E+BRbqfcPUv3YBbCmtvbK1y+sHMI6Z1kXnkX4RoyYiZZXrM680Nh/p0TxNOdNsA==,iv=1h3KbThwTsRaVF+k+dnSwfocSEoyT00X279Dg1Wro60=,aad=foCwpM862VeAD2/7bHRJHAYISneTUJweoSRl2oAdsI4=,tag=tNuCjsNqIy5FVDRu39dQcw==] +- ENC[AES256_GCM,data:v8dfh92oL8IcgjQ=,iv:HgNNPlQh9GNdE+YPvG4Ufpb2I0sIlEpCsOW3lJA1uBE=,aad:21GroP5gb9sCTxZIahN1NhMGqRPQZZksAr5Q7eCeHRc=,tag:gLsjVqot9+Pqck9LJC+bVA==] +- ENC[AES256_GCM,data:X1LMy27AE9SI4h0=,iv:oA1kSg9esGxAvi3qhpcM6Ewrh+p0CFV5cgf6jSPpM08=,aad:CZ7FGJNko6367sd6PwbrIgN/V7Rly4TptbQ1gVsXT1Q=,tag:HerE4nTstX2QZhMn3CPZcw==] +- ENC[AES256_GCM,data:KNkH9iI0bSyvcP3E+BRbqfcPUv3YBbCmtvbK1y+sHMI6Z1kXnkX4RoyYiZZXrM680Nh/p0TxNOdNsA==,iv:1h3KbThwTsRaVF+k+dnSwfocSEoyT00X279Dg1Wro60=,aad:foCwpM862VeAD2/7bHRJHAYISneTUJweoSRl2oAdsI4=,tag:tNuCjsNqIy5FVDRu39dQcw==] sops: kms: - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyGdRODuYMHbA8Ozj8CARCAO7opMolPJUmBXd39Zlp0L2H9fzMKidHm1vvaF6nNFq0ClRY7FlIZmTm4JfnOebPseffiXFn9tG8cq7oi + - enc: CiC6yCOtzsnFhkfdIslYZ0bAf//gYLYCmIu87B3sy/5yYxKnAQEBAQB4usgjrc7JxYZH3SLJWGdGwH//4GC2ApiLvOwd7Mv+cmMAAAB+MHwGCSqGSIb3DQEHBqBvMG0CAQAwaAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyGdRODuYMHbA8Ozj8CARCAO7opMolPJUmBXd39Zlp0L2H9fzMKidHm1vvaF6nNFq0ClRY7FlIZmTm4JfnOebPseffiXFn9tG8cq7oi enc_ts: 1439568549.245995 arn: arn:aws:kms:us-east-1:656532927350:key/920aff2e-c5f1-4040-943a-047fa387b27e diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..c1821d507 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +sops +cryptography>=0.9.3 +boto3>=1.1.3 +ruamel.yaml>=0.10.7 diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..ac56987ed --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from distutils.core import setup + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + +setup( + name = "sops", + packages = ['sops'], + version = "0.2", + author = "Julien Vehent", + author_email = "jvehent@mozilla.com", + description = "Secrets OPerationS (sops) is an editor of encrypted files", + license = "MPL", + keywords = "mozilla secret credential encryption aws kms", + url = "https://github.com/mozilla-services/sops", + long_description= read('README.rst'), + install_requires= ['ruamel.yaml', 'json', 'boto3', 'cryptography'], + classifiers = [ + "Development Status :: 5 - Production/Stable", + "Topic :: Software Development :: Libraries :: Python Modules", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + ], +) diff --git a/sops b/sops index 34d972bbb..a3c47dc30 100755 --- a/sops +++ b/sops @@ -6,51 +6,74 @@ # Contributor: Julien Vehent jvehent@mozilla.com [:ulfr] from __future__ import print_function -import os -import sys -import tempfile -import argparse from base64 import b64encode, b64decode +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, modes, algorithms from ruamel.yaml.comments import CommentedMap from textwrap import dedent -import ruamel.yaml -import json +import argparse import boto3 -from cryptography.hazmat.primitives.ciphers import Cipher, modes, algorithms -from cryptography.hazmat.backends import default_backend -import time -import subprocess +import json +import os import random import re +import ruamel.yaml +import subprocess +import sys +import tempfile +import time DESC = """ `sops` encrypts and decrypts secrets using AWS KMS. It requires access -to AWS uising credentials in ~/.aws/credentials . +to AWS using credentials in ~/.aws/credentials . + +`sops` supports both AWS KMS and PGP encryption. + * To encrypt or decrypt a document with AWS KMS, specify the KMS ARN + in the `-k` flag or in the environment variable SOPS_KMS_ARN. + * To encrypt or decrypt using PGP, specify the PGP fingerprint in the + `-g` flag os in the environment variable SOPS_PGP_FP. + +Those flags are ignored if the document already stores encryption info. +Internally, the KMS and PGP key IDs are stored in the document under +sops.kms and sops.pgp. -The ARN of the KMS Key used to encrypt/decrypt a document must be specified -in a top-level key of the document. For example: YAML sops: kms: - arn: arn:aws:kms:us-east-1:656532927350:key/305caadb - JSON - {"sops":{"kms":{"arn": "arn:aws:kms:us-east-1:650:key/305caadb"}}} - TEXT - SOPS_KMS_ARN="arn:aws:kms:us-east-1:6565350:key/30db-b886-4e12-8d" + - {arn: "aws:kms:us-east-1:656532927350:key/305caadb" } + - {arn: "aws:kms:us-west-2:457153232612:key/f7da420e" } + pgp: + - {fp: 85D77543B3D624B63CEA9E6DBC17301B491B3F21} -Alternatively, the KMS Key ID can be defined on the command line (-k flag) and -in the environment variable $SOPS_KMS_ARN. The order of preference to select -which KMS ID to use is: 1) file, 2) -k flag, 3) environment variable. + JSON + {"sops": { + "kms": [ + {"arn": "aws:kms:us-east-1:650:key/305caadb"}, + {"arn": "aws:kms:us-west-2:457153232612:key/f7da420e" } + ], + "pgp": [ + {"fp": 85D77543B3D624B63CEA9E6DBC17301B491B3F21} + ]} + } + + TEXT (serialized JSON of the `sops` object) + SOPS={"sops":{"kms":[{"arn":"aws:kms:us-east-1:650:ke...}]}} + +The environment variables SOPS_KMS_ARN and SOPS_PGP_FP can take multiple +keys separated by commas. All spaces are trimmed. By default, editing is done in vim. Set the env variable $EDITOR to use a different editor. Mozilla Services - ulfr, relud - 2015 """ + SOPS_KMS_ARN = "" +SOPS_PGP_FP = "" SOPS_FOOTER = "# --- sops encryption info. do not edit. ---" + def main(): argparser = argparse.ArgumentParser( usage='sops ', @@ -60,8 +83,9 @@ def main(): argparser.add_argument('file', help="file to edit; create it if it doesn't exist") argparser.add_argument('-k', '--kms', dest='kmsarn', - help="ARN of KMS key used for (en|de)cryption, " - "if none is defined in .") + help="ARN of KMS key used for encryption") + argparser.add_argument('-g', '--pgp', dest='pgpfp', + help="Fingerprint of PGP key for decryption") argparser.add_argument('-d', '--decrypt', action='store_true', dest='decrypt', help="Decrypt and print it to stdout") @@ -82,6 +106,12 @@ def main(): elif 'SOPS_KMS_ARN' in os.environ: SOPS_KMS_ARN = os.environ['SOPS_KMS_ARN'] + global SOPS_PGP_FP + if args.pgpfp: + SOPS_PGP_FP = args.pgpfp + elif 'SOPS_PGP_FP' in os.environ: + SOPS_PGP_FP = os.environ['SOPS_PGP_FP'] + if args.input_type: itype = args.input_type else: @@ -92,29 +122,31 @@ def main(): else: otype = itype + need_key = False try: fstat = os.stat(args.file) # read the encrypted file from disk - tree = load_tree(args.file, itype) + tree, need_key = load_tree(args.file, itype) except: if args.encrypt or args.decrypt: panic("cannot operate on non-existent file") print("%s doesn't exist, creating it." % args.file) tree = dict() + tree, need_key = verify_or_create_sops_branch(tree) if args.encrypt: # Encrypt mode: encrypt, display and exit - key = get_kms_data_key(tree) + key, tree = get_key(tree, need_key) tree = walk_and_encrypt(tree, key) elif args.decrypt: # Decrypt mode: decrypt, display and exit - key = get_kms_data_key(tree) + key, tree = get_key(tree) tree = walk_and_decrypt(tree, key) else: # EDIT Mode: decrypt, edit, encrypt and save - key = get_kms_data_key(tree) + key, tree = get_key(tree, need_key) # we need a stash to save the IV and AAD and reuse them # if a given value has not changed during editing @@ -134,7 +166,7 @@ def main(): panic("%s has not been modified, exit without writing" % args.file) # encrypt the tree - tree = load_tree(tmppath, otype) + tree, need_key = load_tree(tmppath, otype) os.remove(tmppath) tree = walk_and_encrypt(tree, key, stash) @@ -169,28 +201,56 @@ def load_tree(path, filetype): Read data from `path` using format defined by `filetype`. Return a dictionary with the data """ + tree = dict() with open(path, "r") as fd: if filetype == 'yaml': - return ruamel.yaml.load(fd, ruamel.yaml.RoundTripLoader) + tree = ruamel.yaml.load(fd, ruamel.yaml.RoundTripLoader) elif filetype == 'json': - return json.load(fd) + tree = json.load(fd) else: - tree = dict() - tree['data'] = "" - tree['sops'] = dict() - tree['sops']['kms'] = dict() for line in fd: if line.startswith(SOPS_FOOTER): continue - elif line.startswith('SOPS_KMS_ARN'): - tree['sops']['kms']['arn'] = line.rstrip('\n').split('=', 1)[1] - elif line.startswith('SOPS_KMS_ENCTS'): - tree['sops']['kms']['enc_ts'] = line.rstrip('\n').split('=', 1)[1] - elif line.startswith('SOPS_KMS_ENC'): - tree['sops']['kms']['enc'] = line.rstrip('\n').split('=', 1)[1] + elif line.startswith('SOPS='): + tree['sops'] = json.load( + line.rstrip('\n').split('=', 1)[1]) else: tree['data'] += line - return tree + return verify_or_create_sops_branch(tree) + + +def verify_or_create_sops_branch(tree): + """ + if the current tree doesn't have a sops branch with either kms or pgp + information, create it using the content of the global variables and + indicate that an encryption is needed when returning + """ + if 'sops' not in tree: + tree['sops'] = dict() + if 'kms' in tree['sops'] and isinstance(tree['sops']['kms'], list): + # check that we have at least one ARN to work with + for entry in tree['sops']['kms']: + if 'arn' in entry and entry['arn'] != "": + return tree, False + # if we're here, no arn was found + if 'pgp' in tree['sops'] and isinstance(tree['sops']['pgp'], list): + # check that we have at least one fingerprint to work with + for entry in tree['sops']['pgp']: + if 'fp' in entry and entry['fp'] != "": + return tree, False + # if we're here, no fingerprint was found either + if SOPS_KMS_ARN != "": + tree['sops']['kms'] = list() + for arn in SOPS_KMS_ARN.split(','): + entry = {"arn": arn.replace(" ", "")} + tree['sops']['kms'].append(entry) + if SOPS_PGP_FP != "": + tree['sops']['pgp'] = list() + for fp in SOPS_PGP_FP.split(','): + entry = {"fp": fp.replace(" ", "")} + tree['sops']['pgp'].append(entry) + # return True to indicate an encryption key needs to be created + return tree, True def walk_and_decrypt(branch, key, stash=None): @@ -230,7 +290,7 @@ def decrypt(value, key, stash=None): Return a decrypted value """ # extract fields using a regex - res = re.match(r'^ENC\[AES256_GCM,data=(.+),iv=(.+),aad=(.+),tag=(.+)\]$', + res = re.match(r'^ENC\[AES256_GCM,data:(.+),iv:(.+),aad:(.+),tag:(.+)\]$', value) enc_value = b64decode(res.group(1)) iv = b64decode(res.group(2)) @@ -238,7 +298,8 @@ def decrypt(value, key, stash=None): tag = b64decode(res.group(4)) decryptor = Cipher(algorithms.AES(key), modes.GCM(iv, tag), - default_backend()).decryptor() + default_backend() + ).decryptor() decryptor.authenticate_additional_data(aad) cleartext = decryptor.update(enc_value) + decryptor.finalize() if stash: @@ -300,72 +361,177 @@ def encrypt(value, key, stash=None): default_backend()).encryptor() encryptor.authenticate_additional_data(aad) enc_value = encryptor.update(value) + encryptor.finalize() - return "ENC[AES256_GCM,data={value},iv={iv},aad={aad}," \ - "tag={tag}]".format(value=b64encode(enc_value), + return "ENC[AES256_GCM,data:{value},iv:{iv},aad:{aad}," \ + "tag:{tag}]".format(value=b64encode(enc_value), iv=b64encode(iv), aad=b64encode(aad), tag=b64encode(encryptor.tag)) -def get_kms_arn(tree): +def get_key(tree, need_key=False): """ - Return the ARN of the KMS Key used to encrypt/decrypt the AES key. - If the value exists in the tree, use it. Otherwise try to read it - from env variable SOPS_KMS_ARN. If neither exist, panic! + Obtain a 256 bits symetric key. If the document contain an + encrypted key, try to decrypt it using KMS or PGP. Otherwise, + generate a new random key. """ - if 'sops' not in tree or \ - 'kms' not in tree['sops'] or \ - 'arn' not in tree['sops']['kms'] or \ - tree['sops']['kms']['arn'] == "": - # key is not in tree, try the env variable and store it in the tree - if SOPS_KMS_ARN != "": - if 'sops' not in tree: - tree['sops'] = dict() - if 'kms' not in tree['sops']: - tree['sops']['kms'] = dict() - tree['sops']['kms']['arn'] = SOPS_KMS_ARN - return SOPS_KMS_ARN - else: - panic("KMS ARN not found, unable to continue") - return tree['sops']['kms']['arn'] + if need_key: + # if we're here, the tree doesn't have a key yet. generate + # one and store it in the tree + print("please wait while an encryption key is being generated" + " and stored in a secure fashion", file=sys.stderr) + key = os.urandom(32) + tree = encrypt_key_with_kms(key, tree) + tree = encrypt_key_with_pgp(key, tree) + return key, tree + key = get_key_from_kms(tree) + if not (key is None): + return key, tree + key = get_key_from_pgp(tree) + if not (key is None): + return key, tree + print("[error] couldn't retrieve a key to encrypt/decrypt the tree", + file=sys.stderr) + sys.exit(128) -def get_kms_data_key(tree): - """ - Obtain a key and an IV. The values are either stored in encrypted form in - the tree, and we ask KMS to decrypt it, or it doesn't exist and we - ask KMS to generate one. - Return a 32 bytes key - """ - kms = boto3.client('kms') - kms_key = get_kms_arn(tree) - if 'sops' not in tree or \ - 'kms' not in tree['sops'] or \ - 'enc' not in tree['sops']['kms'] or \ - tree['sops']['kms'] == "": - # no key found, ask KMS for a data key +def get_key_from_kms(tree): + try: + kms_tree = tree['sops']['kms'] + except KeyError: + return None + i = -1 + for entry in kms_tree: + i += 1 try: - kms_response = kms.generate_data_key(KeyId=kms_key, - NumberOfBytes=32) + enc = entry['enc'] + except KeyError: + continue + if 'arn' not in entry or entry['arn'] == "": + print("KMS ARN not found, skipping entry %s" % i, file=sys.stderr) + continue + # extract the region from the ARN + # arn:aws:kms:{REGION}:... + res = re.match(r'^arn:aws:kms:(.+):([0-9]+):key/(.+)$', + entry['arn']) + if res is None: + print("Invalid ARN '%s' in entry %s" % (entry['arn'], i), + file=sys.stderr) + continue + try: + region = res.group(1) except: - panic("Could not obtain data key from KMS using ARN %s" % kms_key) - # store the encrypted blob of the data key in the tree - if 'sops' not in tree: - tree['sops'] = dict() - if 'kms' not in tree['sops']: - tree['sops']['kms'] = dict() - tree['sops']['kms']['enc'] = b64encode(kms_response['CiphertextBlob']) - tree['sops']['kms']['enc_ts'] = time.time() - print("new data key generated from kms: %s..." % - tree['sops']['kms']['enc'][0:32], file=sys.stderr) - else: + print("Unable to find region from ARN '%s' in entry %s" % + (entry['arn'], i), file=sys.stderr) + continue + kms = boto3.client('kms', region_name=region) # use existing data key, ask kms to decrypt it try: - kms_response = kms.decrypt( - CiphertextBlob=b64decode(tree['sops']['kms']['enc'])) + kms_response = kms.decrypt(CiphertextBlob=b64decode(enc)) except Exception as e: - panic("Data key decryption error %s" % e) - return kms_response['Plaintext'][:32] + print("failed to decrypt key using kms: %s, skipping it" % e, + file=sys.stderr) + continue + return kms_response['Plaintext'] + return None + + +def encrypt_key_with_kms(key, tree): + try: + isinstance(tree['sops']['kms'], list) + except KeyError: + return tree + i = -1 + for entry in tree['sops']['kms']: + i += 1 + if 'enc' in entry and entry['enc'] != "": + # key is already encrypted with kms, skipping + continue + if 'arn' not in entry or entry['arn'] == "": + print("KMS ARN not found, skipping entry %d" % i, file=sys.stderr) + continue + arn = entry['arn'] + # extract the region from the ARN + # arn:aws:kms:{REGION}:... + res = re.match(r'^arn:aws:kms:(.+):([0-9]+):key/(.+)$', + arn) + if res is None: + print("Invalid ARN '%s' in entry %s" % (entry['arn'], i), + file=sys.stderr) + continue + try: + region = res.group(1) + except: + print("Unable to find region from ARN '%s' in entry %s" % + (entry['arn'], i), file=sys.stderr) + continue + kms = boto3.client('kms', region_name=region) + try: + kms_response = kms.encrypt(KeyId=arn, Plaintext=key) + except Exception as e: + print("failed to encrypt key using kms arn %s: %s, skipping it" % + (arn, e), file=sys.stderr) + continue + entry['enc'] = b64encode(kms_response['CiphertextBlob']) + entry['created_at'] = time.time() + tree['sops']['kms'][i] = entry + return tree + + +def get_key_from_pgp(tree): + try: + pgp_tree = tree['sops']['pgp'] + except KeyError: + return None + i = -1 + for entry in pgp_tree: + i += 1 + try: + enc = entry['enc'] + except KeyError: + continue + try: + p = subprocess.Popen(['gpg', '-d'], stdout=subprocess.PIPE, + stdin=subprocess.PIPE) + key = p.communicate(input=enc)[0] + except Exception as e: + print("PGP decryption failed in entry %s with error: %s" % + (i, e), file=sys.stderr) + continue + return key + return None + + +def encrypt_key_with_pgp(key, tree): + try: + isinstance(tree['sops']['pgp'], list) + except KeyError: + return tree + i = -1 + for entry in tree['sops']['pgp']: + i += 1 + if 'enc' in entry and entry['enc'] != "": + # key is already encrypted with pgp, skipping + continue + if 'fp' not in entry or entry['fp'] == "": + print("PGP fingerprint not found, skipping entry %d" % i, + file=sys.stderr) + continue + fp = entry['fp'] + try: + p = subprocess.Popen(['gpg', '--no-default-recipient', '--yes', + '--encrypt', '-a', '-r', fp, '--trusted-key', + fp[-16:], '--no-encrypt-to'], + stdout=subprocess.PIPE, + stdin=subprocess.PIPE) + enc = p.communicate(input=key)[0] + except Exception as e: + print("failed to encrypt key using pgp fp %s: %s, skipping it" % + (fp, e), file=sys.stderr) + continue + entry['enc'] = ruamel.yaml.scalarstring.PreservedScalarString(enc) + entry['created_at'] = time.time() + tree['sops']['pgp'][i] = entry + return tree def write_file(tree, path=None, filetype=None): @@ -390,15 +556,10 @@ def write_file(tree, path=None, filetype=None): else: if 'data' in tree: fd.write(tree['data'] + "\n") - if 'sops' in tree and 'kms' in tree['sops']: + if 'sops' in tree: + jsonstr = json.dump(tree['sops']) fd.write("%s\n" % SOPS_FOOTER) - if 'arn' in tree['sops']['kms']: - fd.write("SOPS_KMS_ARN=%s\n" % tree['sops']['kms']['arn']) - if 'enc' in tree['sops']['kms']: - fd.write("SOPS_KMS_ENC=%s\n" % tree['sops']['kms']['enc']) - if 'enc_ts' in tree['sops']['kms']: - fd.write("SOPS_KMS_ENCTS=%s\n" % - tree['sops']['kms']['enc_ts']) + fd.write("SOPS=%s\n" % jsonstr) fd.close() return path