getsops/sops
1
0
Fork 0
mirror of https://github.com/getsops/sops.git synced 2026-02-05 12:45:21 +01:00
Code Issues Activity

Add multidoc encrypt/decrypt for YAML sources

Browse Source
This commit is contained in:
James Robson
2018-09-20 11:18:34 -06:00
parent a12eef6fd9
commit dfa150bf75
19 changed files with 569 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
README.rst
View File

@@ -783,19 +783,18 @@ JSON and TEXT file types do not support anchors and thus have no such limitation
YAML Streams
~~~~~~~~~~~~
``YAML`` supports having more than one document in a single file. ``sops`` does not. For this
reason, the following file won't work in ``sops``:
``YAML`` supports having more than one "document" in a single file, while
formats like ``JSON`` do not. ``sops`` is able to handle both. This means the
following multi-document will be encrypted as expected:
.. code:: yaml
---
data: foo
---
data: bar
If you try to encrypt this file with ``sops``, it will ignore all documents except the first,
effectively deleting them. ``sops`` does not support multi-document files, and until our YAML
parser does, it is unlikely it will.
Note that the ``sops`` metadata, i.e. the hash, etc, is computed for the physical
file rather than each internal "document".
Top-level arrays
~~~~~~~~~~~~~~~~

8
cmd/sops/decrypt.go
View File

@@ -38,7 +38,7 @@ func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
if len(opts.Extract) > 0 {
return extract(tree, opts.Extract, opts.OutputStore)
}
decryptedFile, err = opts.OutputStore.EmitPlainFile(tree.Branch)
decryptedFile, err = opts.OutputStore.EmitPlainFile(tree.Branches)
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
}
@@ -46,13 +46,13 @@ func decrypt(opts decryptOpts) (decryptedFile []byte, err error) {
}
func extract(tree *sops.Tree, path []interface{}, outputStore sops.Store) (output []byte, err error) {
v, err := tree.Branch.Truncate(path)
v, err := tree.Branches[0].Truncate(path)
if err != nil {
return nil, fmt.Errorf("error truncating tree: %s", err)
}
if newBranch, ok := v.(sops.TreeBranch); ok {
tree.Branch = newBranch
decrypted, err := outputStore.EmitPlainFile(tree.Branch)
tree.Branches[0] = newBranch
decrypted, err := outputStore.EmitPlainFile(tree.Branches)
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Error dumping file: %s", err), codes.ErrorDumpingTree)
}

66
cmd/sops/edit.go
View File

@@ -43,30 +43,34 @@ type editExampleOpts struct {
GroupThreshold int
}
var exampleTree = sops.TreeBranch{
sops.TreeItem{
Key: "hello",
Value: `Welcome to SOPS! Edit this file as you please!`,
},
sops.TreeItem{
Key: "example_key",
Value: "example_value",
},
sops.TreeItem{
Key: "example_array",
Value: []interface{}{
"example_value1",
"example_value2",
var exampleTree = sops.Tree{
Branches: sops.TreeBranches{
sops.TreeBranch{
sops.TreeItem{
Key: "hello",
Value: `Welcome to SOPS! Edit this file as you please!`,
},
sops.TreeItem{
Key: "example_key",
Value: "example_value",
},
sops.TreeItem{
Key: "example_array",
Value: []interface{}{
"example_value1",
"example_value2",
},
},
sops.TreeItem{
Key: "example_number",
Value: 1234.56789,
},
sops.TreeItem{
Key: "example_booleans",
Value: []interface{}{true, false},
},
},
},
sops.TreeItem{
Key: "example_number",
Value: 1234.56789,
},
sops.TreeItem{
Key: "example_booleans",
Value: []interface{}{true, false},
},
}
type runEditorUntilOkOpts struct {
@@ -81,16 +85,16 @@ func editExample(opts editExampleOpts) ([]byte, error) {
// Load the example file
var fileBytes []byte
if _, ok := opts.InputStore.(*json.BinaryStore); ok {
// Get the value under the first key
fileBytes = []byte(exampleTree[0].Value.(string))
// Get the value under the first key of the first (possibly only) doc
fileBytes = []byte(exampleTree.Branches[0][0].Value.(string))
} else {
var err error
fileBytes, err = opts.InputStore.EmitPlainFile(exampleTree)
fileBytes, err = opts.InputStore.EmitPlainFile(exampleTree.Branches)
if err != nil {
return nil, err
}
}
branch, err := opts.InputStore.LoadPlainFile(fileBytes)
branches, err := opts.InputStore.LoadPlainFile(fileBytes)
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
}
@@ -99,7 +103,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
return nil, err
}
tree := sops.Tree{
Branch: branch,
Branches: branches,
Metadata: sops.Metadata{
KeyGroups: opts.KeyGroups,
UnencryptedSuffix: opts.UnencryptedSuffix,
@@ -153,7 +157,7 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
if opts.ShowMasterKeys {
out, err = opts.OutputStore.EmitEncryptedFile(*tree)
} else {
out, err = opts.OutputStore.EmitPlainFile(tree.Branch)
out, err = opts.OutputStore.EmitPlainFile(tree.Branches)
}
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
@@ -210,7 +214,7 @@ func runEditorUntilOk(opts runEditorUntilOkOpts) error {
if err != nil {
return common.NewExitError(fmt.Sprintf("Could not read edited file: %s", err), codes.CouldNotReadInputFile)
}
newBranch, err := opts.InputStore.LoadPlainFile(edited)
newBranches, err := opts.InputStore.LoadPlainFile(edited)
if err != nil {
log.WithField(
"error",
@@ -234,11 +238,11 @@ func runEditorUntilOk(opts runEditorUntilOkOpts) error {
bufio.NewReader(os.Stdin).ReadByte()
continue
}
// Replace the whole tree, because otherwise newBranch would
// Replace the whole tree, because otherwise newBranches would
// contain the SOPS metadata
opts.Tree = &t
}
opts.Tree.Branch = newBranch
opts.Tree.Branches = newBranches
needVersionUpdated, err := AIsNewerThanB(version, opts.Tree.Metadata.Version)
if err != nil {
return common.NewExitError(fmt.Sprintf("Failed to compare document version %q with program version %q: %v", opts.Tree.Metadata.Version, version, err), codes.FailedToCompareVersions)

6
cmd/sops/encrypt.go
View File

@@ -58,11 +58,11 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Error reading file: %s", err), codes.CouldNotReadInputFile)
}
branch, err := opts.InputStore.LoadPlainFile(fileBytes)
branches, err := opts.InputStore.LoadPlainFile(fileBytes)
if err != nil {
return nil, common.NewExitError(fmt.Sprintf("Error unmarshalling file: %s", err), codes.CouldNotReadInputFile)
}
if err := ensureNoMetadata(opts, branch); err != nil {
if err := ensureNoMetadata(opts, branches[0]); err != nil {
return nil, common.NewExitError(err, codes.FileAlreadyEncrypted)
}
path, err := filepath.Abs(opts.InputPath)
@@ -70,7 +70,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
return nil, err
}
tree := sops.Tree{
Branch: branch,
Branches: branches,
Metadata: sops.Metadata{
KeyGroups: opts.KeyGroups,
UnencryptedSuffix: opts.UnencryptedSuffix,

2
cmd/sops/main.go
View File

@@ -845,7 +845,7 @@ func jsonValueToTreeInsertableValue(jsonValue string) (interface{}, error) {
return nil, common.NewExitError("Invalid --set value format", codes.ErrorInvalidSetFormat)
}
}
return valueToInsert, nil
return valueToInsert.(sops.TreeBranches)[0], nil
}
func extractSetArguments(set string) (path []interface{}, valueToInsert interface{}, err error) {

2
cmd/sops/set.go
View File

@@ -40,7 +40,7 @@ func set(opts setOpts) ([]byte, error) {
}
// Set the value
tree.Branch = tree.Branch.Set(opts.TreePath, opts.Value)
tree.Branches[0] = tree.Branches[0].Set(opts.TreePath, opts.Value)
err = common.EncryptTree(common.EncryptTreeOpts{
DataKey: dataKey, Tree: tree, Cipher: opts.Cipher,

4
decrypt/decrypt.go
View File

@@ -11,9 +11,9 @@ import (
"go.mozilla.org/sops"
"go.mozilla.org/sops/aes"
sopsdotenv "go.mozilla.org/sops/stores/dotenv"
sopsjson "go.mozilla.org/sops/stores/json"
sopsyaml "go.mozilla.org/sops/stores/yaml"
sopsdotenv "go.mozilla.org/sops/stores/dotenv"
)
// File is a wrapper around Data that reads a local encrypted
@@ -73,5 +73,5 @@ func Data(data []byte, format string) (cleartext []byte, err error) {
return nil, fmt.Errorf("Failed to verify data integrity. expected mac %q, got %q", originalMac, mac)
}
return store.EmitPlainFile(tree.Branch)
return store.EmitPlainFile(tree.Branches)
}

183
sops.go
View File

@@ -100,6 +100,9 @@ type TreeItem struct {
// TreeBranch is a branch inside sops's tree. It is a slice of TreeItems and is therefore ordered
type TreeBranch []TreeItem
// Trees usually have more than one branch
type TreeBranches []TreeBranch
func valueFromPathAndLeaf(path []interface{}, leaf interface{}) interface{} {
switch component := path[0].(type) {
case int:
@@ -178,8 +181,8 @@ func (branch TreeBranch) Set(path []interface{}, value interface{}) TreeBranch {
// Tree is the data structure used by sops to represent documents internally
type Tree struct {
Branch TreeBranch
Metadata Metadata
Branches TreeBranches
// FilePath is the path of the file this struct represents
FilePath string
}
@@ -284,52 +287,62 @@ func (branch TreeBranch) walkBranch(in TreeBranch, path []string, onLeaves func(
return in, nil
}
// Encrypt walks over the tree and encrypts all values with the provided cipher, except those whose key ends with the UnencryptedSuffix specified on the Metadata struct, or those not ending with EncryptedSuffix, if EncryptedSuffix is provided (by default it is not).
// If encryption is successful, it returns the MAC for the encrypted tree.
// Encrypt walks over the tree and encrypts all values with the provided cipher,
// except those whose key ends with the UnencryptedSuffix specified on the
// Metadata struct, or those not ending with EncryptedSuffix, if EncryptedSuffix
// is provided (by default it is not). If encryption is successful, it returns
// the MAC for the encrypted tree.
func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) {
audit.SubmitEvent(audit.EncryptEvent{
File: tree.FilePath,
})
hash := sha512.New()
_, err := tree.Branch.walkBranch(tree.Branch, make([]string, 0), func(in interface{}, path []string) (interface{}, error) {
// Only add to MAC if not a comment
if _, ok := in.(Comment); !ok {
bytes, err := ToBytes(in)
if err != nil {
return nil, fmt.Errorf("Could not convert %s to bytes: %s", in, err)
walk := func(branch TreeBranch) error {
_, err := branch.walkBranch(branch, make([]string, 0), func(in interface{}, path []string) (interface{}, error) {
// Only add to MAC if not a comment
if _, ok := in.(Comment); !ok {
bytes, err := ToBytes(in)
if err != nil {
return nil, fmt.Errorf("Could not convert %s to bytes: %s", in, err)
}
hash.Write(bytes)
}
hash.Write(bytes)
}
encrypted := true
if tree.Metadata.UnencryptedSuffix != "" {
for _, v := range path {
if strings.HasSuffix(v, tree.Metadata.UnencryptedSuffix) {
encrypted = false
break
encrypted := true
if tree.Metadata.UnencryptedSuffix != "" {
for _, v := range path {
if strings.HasSuffix(v, tree.Metadata.UnencryptedSuffix) {
encrypted = false
break
}
}
}
}
if tree.Metadata.EncryptedSuffix != "" {
encrypted = false
for _, v := range path {
if strings.HasSuffix(v, tree.Metadata.EncryptedSuffix) {
encrypted = true
break
if tree.Metadata.EncryptedSuffix != "" {
encrypted = false
for _, v := range path {
if strings.HasSuffix(v, tree.Metadata.EncryptedSuffix) {
encrypted = true
break
}
}
}
}
if encrypted {
var err error
pathString := strings.Join(path, ":") + ":"
in, err = cipher.Encrypt(in, key, pathString)
if err != nil {
return nil, fmt.Errorf("Could not encrypt value: %s", err)
if encrypted {
var err error
pathString := strings.Join(path, ":") + ":"
in, err = cipher.Encrypt(in, key, pathString)
if err != nil {
return nil, fmt.Errorf("Could not encrypt value: %s", err)
}
}
return in, nil
})
return err
}
for _, branch := range tree.Branches {
err := walk(branch)
if err != nil {
return "", fmt.Errorf("Error walking tree: %s", err)
}
return in, nil
})
if err != nil {
return "", fmt.Errorf("Error walking tree: %s", err)
}
return fmt.Sprintf("%X", hash.Sum(nil)), nil
}
@@ -342,61 +355,67 @@ func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) {
File: tree.FilePath,
})
hash := sha512.New()
_, err := tree.Branch.walkBranch(tree.Branch, make([]string, 0), func(in interface{}, path []string) (interface{}, error) {
encrypted := true
if tree.Metadata.UnencryptedSuffix != "" {
for _, p := range path {
if strings.HasSuffix(p, tree.Metadata.UnencryptedSuffix) {
encrypted = false
break
walk := func(branch TreeBranch) error {
_, err := branch.walkBranch(branch, make([]string, 0), func(in interface{}, path []string) (interface{}, error) {
encrypted := true
if tree.Metadata.UnencryptedSuffix != "" {
for _, p := range path {
if strings.HasSuffix(p, tree.Metadata.UnencryptedSuffix) {
encrypted = false
break
}
}
}
}
if tree.Metadata.EncryptedSuffix != "" {
encrypted = false
for _, p := range path {
if strings.HasSuffix(p, tree.Metadata.EncryptedSuffix) {
encrypted = true
break
if tree.Metadata.EncryptedSuffix != "" {
encrypted = false
for _, p := range path {
if strings.HasSuffix(p, tree.Metadata.EncryptedSuffix) {
encrypted = true
break
}
}
}
}
var v interface{}
if encrypted {
var err error
pathString := strings.Join(path, ":") + ":"
if c, ok := in.(Comment); ok {
v, err = cipher.Decrypt(c.Value, key, pathString)
if err != nil {
// Assume the comment was not encrypted in the first place
log.WithField("comment", c.Value).
Warn("Found possibly unencrypted comment in file. " +
"This is to be expected if the file being " +
"decrypted was created with an older version of " +
"SOPS.")
v = c
var v interface{}
if encrypted {
var err error
pathString := strings.Join(path, ":") + ":"
if c, ok := in.(Comment); ok {
v, err = cipher.Decrypt(c.Value, key, pathString)
if err != nil {
// Assume the comment was not encrypted in the first place
log.WithField("comment", c.Value).
Warn("Found possibly unencrypted comment in file. " +
"This is to be expected if the file being " +
"decrypted was created with an older version of " +
"SOPS.")
v = c
}
} else {
v, err = cipher.Decrypt(in.(string), key, pathString)
if err != nil {
return nil, fmt.Errorf("Could not decrypt value: %s", err)
}
}
} else {
v, err = cipher.Decrypt(in.(string), key, pathString)
v = in
}
// Only add to MAC if not a comment
if _, ok := v.(Comment); !ok {
bytes, err := ToBytes(v)
if err != nil {
return nil, fmt.Errorf("Could not decrypt value: %s", err)
return nil, fmt.Errorf("Could not convert %s to bytes: %s", in, err)
}
hash.Write(bytes)
}
} else {
v = in
return v, nil
})
return err
}
for _, branch := range tree.Branches {
err := walk(branch)
if err != nil {
return "", fmt.Errorf("Error walking tree: %s", err)
}
// Only add to MAC if not a comment
if _, ok := v.(Comment); !ok {
bytes, err := ToBytes(v)
if err != nil {
return nil, fmt.Errorf("Could not convert %s to bytes: %s", in, err)
}
hash.Write(bytes)
}
return v, nil
})
if err != nil {
return "", fmt.Errorf("Error walking tree: %s", err)
}
return fmt.Sprintf("%X", hash.Sum(nil)), nil
}
@@ -451,7 +470,7 @@ type EncryptedFileLoader interface {
// way to load unencrypted files into SOPS. Because the files it loads are
// unencrypted, the returned data structure does not contain any metadata.
type PlainFileLoader interface {
LoadPlainFile(in []byte) (TreeBranch, error)
LoadPlainFile(in []byte) (TreeBranches, error)
}
// EncryptedFileEmitter is the interface for emitting encrypting files. It provides a
@@ -464,7 +483,7 @@ type EncryptedFileEmitter interface {
// to emit plain text files from the internal SOPS representation so that they can be
// shown
type PlainFileEmitter interface {
EmitPlainFile(TreeBranch) ([]byte, error)
EmitPlainFile(TreeBranches) ([]byte, error)
}
type ValueEmitter interface {

410
sops_test.go
View File

@@ -35,22 +35,24 @@ func (c reverseCipher) Decrypt(value string, key []byte, path string) (plaintext
}
func TestUnencryptedSuffix(t *testing.T) {
branch := TreeBranch{
TreeItem{
Key: "foo_unencrypted",
Value: "bar",
},
TreeItem{
Key: "bar_unencrypted",
Value: TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
branches := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo_unencrypted",
Value: "bar",
},
TreeItem{
Key: "bar_unencrypted",
Value: TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
},
},
},
}
tree := Tree{Branch: branch, Metadata: Metadata{UnencryptedSuffix: "_unencrypted"}}
tree := Tree{Branches: branches, Metadata: Metadata{UnencryptedSuffix: "_unencrypted"}}
expected := TreeBranch{
TreeItem{
Key: "foo_unencrypted",
@@ -71,35 +73,37 @@ func TestUnencryptedSuffix(t *testing.T) {
if err != nil {
t.Errorf("Encrypting the tree failed: %s", err)
}
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches[0], expected) {
t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branches[0], expected)
}
_, err = tree.Decrypt(bytes.Repeat([]byte("f"), 32), cipher)
if err != nil {
t.Errorf("Decrypting the tree failed: %s", err)
}
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches[0], expected) {
t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branches[0], expected)
}
}
func TestEncryptedSuffix(t *testing.T) {
branch := TreeBranch{
TreeItem{
Key: "foo_encrypted",
Value: "bar",
},
TreeItem{
Key: "bar",
Value: TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
branches := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo_encrypted",
Value: "bar",
},
TreeItem{
Key: "bar",
Value: TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
},
},
},
}
tree := Tree{Branch: branch, Metadata: Metadata{EncryptedSuffix: "_encrypted"}}
tree := Tree{Branches: branches, Metadata: Metadata{EncryptedSuffix: "_encrypted"}}
expected := TreeBranch{
TreeItem{
Key: "foo_encrypted",
@@ -120,16 +124,16 @@ func TestEncryptedSuffix(t *testing.T) {
if err != nil {
t.Errorf("Encrypting the tree failed: %s", err)
}
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches[0], expected) {
t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branches[0], expected)
}
_, err = tree.Decrypt(bytes.Repeat([]byte("f"), 32), cipher)
if err != nil {
t.Errorf("Decrypting the tree failed: %s", err)
}
expected[0].Value = "bar"
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches[0], expected) {
t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branches[0], expected)
}
}
@@ -144,126 +148,200 @@ func (m MockCipher) Decrypt(value string, key []byte, path string) (interface{},
}
func TestEncrypt(t *testing.T) {
branch := TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: 5,
branches := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: 5,
},
},
},
},
TreeItem{
Key: "bar",
Value: false,
},
TreeItem{
Key: "foobar",
Value: 2.12,
},
TreeItem{
Key: "barfoo",
Value: nil,
},
}
expected := TreeBranch{
TreeItem{
Key: "foo",
Value: "a",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "a",
},
TreeItem{
Key: "bar",
Value: false,
},
TreeItem{
Key: "foobar",
Value: 2.12,
},
TreeItem{
Key: "barfoo",
Value: nil,
},
},
TreeItem{
Key: "bar",
Value: "a",
TreeBranch{
TreeItem{
Key: "foo2",
Value: "bar",
},
},
TreeItem{
Key: "foobar",
Value: "a",
},
TreeItem{
Key: "barfoo",
Value: nil,
TreeBranch{
TreeItem{
Key: "foo3",
Value: "bar",
},
},
}
tree := Tree{Branch: branch, Metadata: Metadata{UnencryptedSuffix: DefaultUnencryptedSuffix}}
expected := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo",
Value: "a",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "a",
},
},
},
TreeItem{
Key: "bar",
Value: "a",
},
TreeItem{
Key: "foobar",
Value: "a",
},
TreeItem{
Key: "barfoo",
Value: nil,
},
},
TreeBranch{
TreeItem{
Key: "foo2",
Value: "a",
},
},
TreeBranch{
TreeItem{
Key: "foo3",
Value: "a",
},
},
}
tree := Tree{Branches: branches, Metadata: Metadata{UnencryptedSuffix: DefaultUnencryptedSuffix}}
tree.Encrypt(bytes.Repeat([]byte{'f'}, 32), MockCipher{})
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("%s does not equal expected tree: %s", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches, expected) {
t.Errorf("%s does not equal expected tree: %s", tree.Branches, expected)
}
}
func TestDecrypt(t *testing.T) {
branch := TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
branches := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "5",
},
},
},
TreeItem{
Key: "bar",
Value: "false",
},
TreeItem{
Key: "foobar",
Value: "2.12",
},
TreeItem{
Key: "barfoo",
Value: nil,
},
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "5",
TreeBranch{
TreeItem{
Key: "foo",
Value: "bar",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "6",
},
},
},
},
TreeItem{
Key: "bar",
Value: "false",
},
TreeItem{
Key: "foobar",
Value: "2.12",
},
TreeItem{
Key: "barfoo",
Value: nil,
TreeBranch{
TreeItem{
Key: "foo3",
Value: "bar",
},
},
}
expected := TreeBranch{
TreeItem{
Key: "foo",
Value: "a",
expected := TreeBranches{
TreeBranch{
TreeItem{
Key: "foo",
Value: "a",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "a",
},
},
},
TreeItem{
Key: "bar",
Value: "a",
},
TreeItem{
Key: "foobar",
Value: "a",
},
TreeItem{
Key: "barfoo",
Value: nil,
},
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "a",
TreeBranch{
TreeItem{
Key: "foo",
Value: "a",
},
TreeItem{
Key: "baz",
Value: TreeBranch{
TreeItem{
Key: "bar",
Value: "a",
},
},
},
},
TreeItem{
Key: "bar",
Value: "a",
},
TreeItem{
Key: "foobar",
Value: "a",
},
TreeItem{
Key: "barfoo",
Value: nil,
TreeBranch{
TreeItem{
Key: "foo3",
Value: "a",
},
},
}
tree := Tree{Branch: branch, Metadata: Metadata{UnencryptedSuffix: DefaultUnencryptedSuffix}}
tree := Tree{Branches: branches, Metadata: Metadata{UnencryptedSuffix: DefaultUnencryptedSuffix}}
tree.Decrypt(bytes.Repeat([]byte{'f'}, 32), MockCipher{})
if !reflect.DeepEqual(tree.Branch, expected) {
t.Errorf("%s does not equal expected tree: %s", tree.Branch, expected)
if !reflect.DeepEqual(tree.Branches, expected) {
t.Errorf("%s does not equal expected tree: %s", tree.Branches[0], expected)
}
}
@@ -300,17 +378,19 @@ func TestTruncateTree(t *testing.T) {
func TestEncryptComments(t *testing.T) {
tree := Tree{
Branch: TreeBranch{
TreeItem{
Key: Comment{"foo"},
Value: nil,
},
TreeItem{
Key: "list",
Value: []interface{}{
"1",
Comment{"bar"},
"2",
Branches: TreeBranches{
TreeBranch{
TreeItem{
Key: Comment{"foo"},
Value: nil,
},
TreeItem{
Key: "list",
Value: []interface{}{
"1",
Comment{"bar"},
"2",
},
},
},
},
@@ -319,28 +399,30 @@ func TestEncryptComments(t *testing.T) {
},
}
tree.Encrypt(bytes.Repeat([]byte{'f'}, 32), reverseCipher{})
assert.Equal(t, "oof", tree.Branch[0].Key.(Comment).Value)
assert.Equal(t, "rab", tree.Branch[1].Value.([]interface{})[1])
assert.Equal(t, "oof", tree.Branches[0][0].Key.(Comment).Value)
assert.Equal(t, "rab", tree.Branches[0][1].Value.([]interface{})[1])
}
func TestDecryptComments(t *testing.T) {
tree := Tree{
Branch: TreeBranch{
TreeItem{
Key: Comment{"oof"},
Value: nil,
},
TreeItem{
Key: "list",
Value: []interface{}{
"1",
Comment{"rab"},
"2",
Branches: TreeBranches{
TreeBranch{
TreeItem{
Key: Comment{"oof"},
Value: nil,
},
TreeItem{
Key: "list",
Value: []interface{}{
"1",
Comment{"rab"},
"2",
},
},
TreeItem{
Key: "list",
Value: nil,
},
},
TreeItem{
Key: "list",
Value: nil,
},
},
Metadata: Metadata{
@@ -348,23 +430,25 @@ func TestDecryptComments(t *testing.T) {
},
}
tree.Decrypt(bytes.Repeat([]byte{'f'}, 32), reverseCipher{})
assert.Equal(t, "foo", tree.Branch[0].Key.(Comment).Value)
assert.Equal(t, "bar", tree.Branch[1].Value.([]interface{})[1])
assert.Equal(t, "foo", tree.Branches[0][0].Key.(Comment).Value)
assert.Equal(t, "bar", tree.Branches[0][1].Value.([]interface{})[1])
}
func TestDecryptUnencryptedComments(t *testing.T) {
tree := Tree{
Branch: TreeBranch{
TreeItem{
// We use `error` to simulate an error decrypting, the fake cipher will error in this case
Key: Comment{"error"},
Value: nil,
Branches: TreeBranches{
TreeBranch{
TreeItem{
// We use `error` to simulate an error decrypting, the fake cipher will error in this case
Key: Comment{"error"},
Value: nil,
},
},
},
Metadata: Metadata{},
}
tree.Decrypt(bytes.Repeat([]byte{'f'}, 32), reverseCipher{})
assert.Equal(t, "error", tree.Branch[0].Key.(Comment).Value)
assert.Equal(t, "error", tree.Branches[0][0].Key.(Comment).Value)
}
func TestSetNewKey(t *testing.T) {

23
stores/dotenv/store.go
View File

@@ -17,14 +17,14 @@ type Store struct {
}
func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
branch, err := store.LoadPlainFile(in)
branches, err := store.LoadPlainFile(in)
if err != nil {
return sops.Tree{}, err
}
var resultBranch sops.TreeBranch
mdMap := make(map[string]interface{})
for _, item := range branch {
for _, item := range branches[0] {
s := item.Key.(string)
if strings.HasPrefix(s, SopsPrefix) {
s = s[len(SopsPrefix):]
@@ -44,12 +44,15 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
}
return sops.Tree{
Branch: resultBranch,
Branches: sops.TreeBranches{
resultBranch,
},
Metadata: internalMetadata,
}, nil
}
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranch, error) {
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
var branches sops.TreeBranches
var branch sops.TreeBranch
for _, line := range bytes.Split(in, []byte("\n")) {
@@ -65,7 +68,9 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranch, error) {
Value: string(line[pos+1:]),
})
}
return branch, nil
branches = append(branches, branch)
return branches, nil
}
func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
@@ -78,14 +83,14 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
if value == nil {
continue
}
in.Branch = append(in.Branch, sops.TreeItem{Key: SopsPrefix + key, Value: value})
in.Branches[0] = append(in.Branches[0], sops.TreeItem{Key: SopsPrefix + key, Value: value})
}
return store.EmitPlainFile(in.Branch)
return store.EmitPlainFile(in.Branches)
}
func (store *Store) EmitPlainFile(in sops.TreeBranch) ([]byte, error) {
func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) {
buffer := bytes.Buffer{}
for _, item := range in {
for _, item := range in[0] {
if isComplexValue(item.Value) {
return nil, fmt.Errorf("cannot use complex value in dotenv file: %s", item.Value)
}

9
stores/dotenv/store_test.go
View File

@@ -30,12 +30,15 @@ var BRANCH = sops.TreeBranch{
}
func TestLoadPlainFile(t *testing.T) {
branch, err := (&Store{}).LoadPlainFile(PLAIN)
branches, err := (&Store{}).LoadPlainFile(PLAIN)
assert.Nil(t, err)
assert.Equal(t, BRANCH, branch)
assert.Equal(t, BRANCH, branches[0])
}
func TestEmitPlainFile(t *testing.T) {
bytes, err := (&Store{}).EmitPlainFile(BRANCH)
branches := sops.TreeBranches{
BRANCH,
}
bytes, err := (&Store{}).EmitPlainFile(branches)
assert.Nil(t, err)
assert.Equal(t, PLAIN, bytes)
}

35
stores/json/store.go
View File

@@ -23,11 +23,13 @@ func (store BinaryStore) LoadEncryptedFile(in []byte) (sops.Tree, error) {
return store.store.LoadEncryptedFile(in)
}
func (store BinaryStore) LoadPlainFile(in []byte) (sops.TreeBranch, error) {
return sops.TreeBranch{
sops.TreeItem{
Key: "data",
Value: string(in),
func (store BinaryStore) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
return sops.TreeBranches{
sops.TreeBranch{
sops.TreeItem{
Key: "data",
Value: string(in),
},
},
}, nil
}
@@ -36,8 +38,9 @@ func (store BinaryStore) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
return store.store.EmitEncryptedFile(in)
}
func (store BinaryStore) EmitPlainFile(in sops.TreeBranch) ([]byte, error) {
for _, item := range in {
func (store BinaryStore) EmitPlainFile(in sops.TreeBranches) ([]byte, error) {
// JSON stores a single object per file
for _, item := range in[0] {
if item.Key == "data" {
return []byte(item.Value.(string)), nil
}
@@ -238,21 +241,25 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
}
}
return sops.Tree{
Branch: branch,
Branches: sops.TreeBranches{
branch,
},
Metadata: metadata,
}, nil
}
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranch, error) {
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
branch, err := store.treeBranchFromJSON(in)
if err != nil {
return branch, fmt.Errorf("Could not unmarshal input data: %s", err)
return nil, fmt.Errorf("Could not unmarshal input data: %s", err)
}
return branch, nil
return sops.TreeBranches{
branch,
}, nil
}
func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
tree := append(in.Branch, sops.TreeItem{Key: "sops", Value: stores.MetadataFromInternal(in.Metadata)})
tree := append(in.Branches[0], sops.TreeItem{Key: "sops", Value: stores.MetadataFromInternal(in.Metadata)})
out, err := store.jsonFromTreeBranch(tree)
if err != nil {
return nil, fmt.Errorf("Error marshaling to json: %s", err)
@@ -260,8 +267,8 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
return out, nil
}
func (store *Store) EmitPlainFile(in sops.TreeBranch) ([]byte, error) {
out, err := store.jsonFromTreeBranch(in)
func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) {
out, err := store.jsonFromTreeBranch(in[0])
if err != nil {
return nil, fmt.Errorf("Error marshaling to json: %s", err)
}

42
stores/json/store_test.go
View File

@@ -241,21 +241,25 @@ func TestEncodeJSONWithEscaping(t *testing.T) {
}
func TestEncodeJSONArrayOfObjects(t *testing.T) {
branch := sops.TreeBranch{
sops.TreeItem{
Key: "foo",
Value: []interface{}{
sops.TreeBranch{
sops.TreeItem{
Key: "foo",
Value: 3,
},
sops.TreeItem{
Key: "bar",
Value: false,
tree := sops.Tree{
Branches: sops.TreeBranches{
sops.TreeBranch{
sops.TreeItem{
Key: "foo",
Value: []interface{}{
sops.TreeBranch{
sops.TreeItem{
Key: "foo",
Value: 3,
},
sops.TreeItem{
Key: "bar",
Value: false,
},
},
2,
},
},
2,
},
},
}
@@ -269,7 +273,7 @@ func TestEncodeJSONArrayOfObjects(t *testing.T) {
]
}`
store := Store{}
out, err := store.EmitPlainFile(branch)
out, err := store.EmitPlainFile(tree.Branches)
assert.Nil(t, err)
assert.Equal(t, expected, string(out))
}
@@ -286,7 +290,13 @@ func TestLoadJSONFormattedBinaryFile(t *testing.T) {
// e.g. because the --input-type binary flag was provided.
data := []byte(`{"hello": 2}`)
store := BinaryStore{}
branch, err := store.LoadPlainFile(data)
branches, err := store.LoadPlainFile(data)
assert.Nil(t, err)
assert.Equal(t, "data", branch[0].Key)
assert.Equal(t, "data", branches[0][0].Key)
}
func TestEmitValueString(t *testing.T) {
bytes, err := (&Store{}).EmitValue("hello")
assert.Nil(t, err)
assert.Equal(t, []byte("\"hello\""), bytes)
}

69
stores/yaml/store.go
View File

@@ -102,6 +102,10 @@ func (store Store) treeBranchToYamlMap(in sops.TreeBranch) yaml.MapSlice {
}
func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
var data []yaml.MapSlice
if err := (yaml.CommentUnmarshaler{}).UnmarshalDocuments(in, &data); err != nil {
return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err)
}
// Because we don't know what fields the input file will have, we have to
// load the file in two steps.
// First, we load the file's metadata, the structure of which is known.
@@ -117,46 +121,63 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
if err != nil {
return sops.Tree{}, err
}
// After that, we load the whole file into a map.
var data yaml.MapSlice
if err := (yaml.CommentUnmarshaler{}).Unmarshal(in, &data); err != nil {
return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err)
}
// Discard metadata, as we already loaded it.
for i, item := range data {
if item.Key == "sops" {
data = append(data[:i], data[i+1:]...)
var branches sops.TreeBranches
for _, doc := range data {
for i, item := range doc {
if item.Key == "sops" { // Erase
doc = append(doc[:i], doc[i+1:]...)
}
}
branches = append(branches, store.mapSliceToTreeBranch(doc))
}
return sops.Tree{
Branch: store.mapSliceToTreeBranch(data),
Branches: branches,
Metadata: metadata,
}, nil
}
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranch, error) {
var data yaml.MapSlice
if err := (yaml.CommentUnmarshaler{}).Unmarshal(in, &data); err != nil {
func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
var data []yaml.MapSlice
if err := (yaml.CommentUnmarshaler{}).UnmarshalDocuments(in, &data); err != nil {
return nil, fmt.Errorf("Error unmarshaling input YAML: %s", err)
}
return store.mapSliceToTreeBranch(data), nil
var branches sops.TreeBranches
for _, doc := range data {
branches = append(branches, store.mapSliceToTreeBranch(doc))
}
return branches, nil
}
func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
yamlMap := store.treeBranchToYamlMap(in.Branch)
yamlMap = append(yamlMap, yaml.MapItem{Key: "sops", Value: stores.MetadataFromInternal(in.Metadata)})
out, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap)
if err != nil {
return nil, fmt.Errorf("Error marshaling to yaml: %s", err)
out := []byte{}
for i, branch := range in.Branches {
if i > 0 {
out = append(out, "---\n"...)
}
yamlMap := store.treeBranchToYamlMap(branch)
yamlMap = append(yamlMap, yaml.MapItem{Key: "sops", Value: stores.MetadataFromInternal(in.Metadata)})
tout, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap)
if err != nil {
return nil, fmt.Errorf("Error marshaling to yaml: %s", err)
}
out = append(out, tout...)
}
return out, nil
}
func (store *Store) EmitPlainFile(in sops.TreeBranch) ([]byte, error) {
yamlMap := store.treeBranchToYamlMap(in)
out, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap)
if err != nil {
return nil, fmt.Errorf("Error marshaling to yaml: %s", err)
func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) {
var out []byte
for i, branch := range branches {
if i > 0 {
out = append(out, "---\n"...)
}
yamlMap := store.treeBranchToYamlMap(branch)
tmpout, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap)
if err != nil {
return nil, fmt.Errorf("Error marshaling to yaml: %s", err)
}
out = append(out[:], tmpout[:]...)
}
return out, nil
}

37
stores/yaml/store_test.go
View File

@@ -7,8 +7,45 @@ import (
"go.mozilla.org/sops"
)
var PLAIN = []byte(`---
# comment 0
key1: value
key1_a: value
# ^ comment 1
---
key2: value2`)
var BRANCHES = sops.TreeBranches{
sops.TreeBranch{
sops.TreeItem{
Key: "key1",
Value: "value",
},
sops.TreeItem{
Key: "key1_a",
Value: "value",
},
sops.TreeItem{
Key: sops.Comment{" ^ comment 1"},
Value: nil,
},
},
sops.TreeBranch{
sops.TreeItem{
Key: "key2",
Value: "value2",
},
},
}
func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) {
data := []byte(`hello: 2`)
_, err := (&Store{}).LoadEncryptedFile(data)
assert.Equal(t, sops.MetadataNotFound, err)
}
func TestLoadPlainFile(t *testing.T) {
branches, err := (&Store{}).LoadPlainFile(PLAIN)
assert.Nil(t, err)
assert.Equal(t, BRANCHES, branches)
}

15
vendor/github.com/mozilla-services/yaml/decode_test.go generated vendored
View File

@@ -1005,6 +1005,21 @@ func (s *S) TestUnmarshalDocuments(c *C) {
{{"c", []interface{}{"j"}}, {"d", yaml.MapSlice{{"e", "f"}}}},
},
},
{ // Buffer of exactly 512 (input_raw_buffer_size) bytes
"key1: 012345678901234567890123456789012345678901234567890123456789012345678901234567893456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901\n---\nkey2: value",
[]map[string]string{
{"key1": "012345678901234567890123456789012345678901234567890123456789012345678901234567893456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901"},
{"key2": "value"},
},
},
{ // Buffer over 512 (input_raw_buffer_size) bytes
"key1: 01234567890123456789012345678901234567890123456789012345678901234567890123456789345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789\n---\nkey2: value\n---\nkey3: value",
[]map[string]string{
{"key1": "01234567890123456789012345678901234567890123456789012345678901234567890123456789345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"},
{"key2": "value"},
{"key3": "value"},
},
},
}
for _, item := range docTests {
v := reflect.ValueOf(item.value)

4
vendor/github.com/mozilla-services/yaml/encode_test.go generated vendored
View File

@@ -3,14 +3,14 @@ package yaml_test
import (
"fmt"
"math"
"net"
"os"
"strconv"
"strings"
"time"
"go.mozilla.org/yaml.v2"
. "gopkg.in/check.v1"
"net"
"os"
)
var marshalIntTest = 123

5
vendor/github.com/mozilla-services/yaml/readerc.go generated vendored
View File

@@ -4,13 +4,14 @@ import (
"io"
)
func yaml_parser_remaining_buffer(parser *yaml_parser_t) []byte {
func yaml_parser_remaining_bytes(parser *yaml_parser_t) []byte {
if parser.unread > 0 {
offset := len(parser.buffer) - parser.unread
if int(parser.buffer[offset]) == 0 {
return []byte{}
}
return parser.buffer[offset:]
remaining := append(parser.buffer[offset:], parser.input[parser.input_pos:]...)
return remaining
}
return []byte{}
}

2
vendor/github.com/mozilla-services/yaml/yaml.go generated vendored
View File

@@ -98,7 +98,7 @@ func unmarshalDocuments(in []byte, out interface{}, withComments bool) (err erro
p.destroy()
return &TypeError{d.terrors}
}
in = yaml_parser_remaining_buffer(&p.parser)
in = yaml_parser_remaining_bytes(&p.parser)
p.destroy()
if len(in) == 0 {
break