diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 629de5fe2..84c10bc85 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -13,6 +13,7 @@ import ( "go.mozilla.org/sops/stores/json" "go.mozilla.org/sops/stores/yaml" "go.mozilla.org/sops/stores/dotenv" + "go.mozilla.org/sops/stores/ini" "gopkg.in/urfave/cli.v1" ) @@ -109,6 +110,10 @@ func IsEnvFile(path string) bool { return strings.HasSuffix(path, ".env") } +func IsIniFile(path string) bool { + return strings.HasSuffix(path, ".ini") +} + func DefaultStoreForPath(path string) sops.Store { if IsYAMLFile(path) { return &yaml.Store{} @@ -116,6 +121,8 @@ func DefaultStoreForPath(path string) sops.Store { return &json.Store{} } else if IsEnvFile(path) { return &dotenv.Store{} + } else if IsIniFile(path) { + return &ini.Store{} } return &json.BinaryStore{} } diff --git a/cmd/sops/main.go b/cmd/sops/main.go index f7bc2665b..31b3720c8 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -33,6 +33,7 @@ import ( yamlstores "go.mozilla.org/sops/stores/yaml" "google.golang.org/grpc" "gopkg.in/urfave/cli.v1" + "go.mozilla.org/sops/stores/ini" ) var log *logrus.Logger @@ -690,6 +691,8 @@ func inputStore(context *cli.Context, path string) sops.Store { return &json.Store{} case "dotenv": return &dotenv.Store{} + case "ini": + return &ini.Store{} case "binary": return &json.BinaryStore{} default: @@ -705,6 +708,8 @@ func outputStore(context *cli.Context, path string) sops.Store { return &json.Store{} case "dotenv": return &dotenv.Store{} + case "ini": + return &ini.Store{} case "binary": return &json.BinaryStore{} default: diff --git a/stores/ini/store.go b/stores/ini/store.go new file mode 100644 index 000000000..4d2b206aa --- /dev/null +++ b/stores/ini/store.go @@ -0,0 +1,369 @@ +package ini //import "go.mozilla.org/sops/stores/ini" + +import ( + "bytes" + "encoding/json" + "fmt" + + "go.mozilla.org/sops" + "go.mozilla.org/sops/stores" + "strconv" + "reflect" + "gopkg.in/ini.v1" + "strings" + "sort" +) + +// Store handles storage of ini data. +type Store struct { +} + +func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) { + iniFile := ini.Empty() + for _, branch := range branches { + for _, item := range branch { + if _, ok := item.Key.(sops.Comment); ok { + continue + } + section, err := iniFile.NewSection(item.Key.(string)) + if err != nil { + return nil, fmt.Errorf("Error encoding section %s: %s", item.Key, err) + } + itemTree, ok := item.Value.(sops.TreeBranch) + if !ok { + return nil, fmt.Errorf("Error encoding section: Section values should always be TreeBranches") + } + + first := 0 + if len(itemTree) > 0 { + if sectionComment, ok := itemTree[0].Key.(sops.Comment); ok { + section.Comment = sectionComment.Value + first = 1 + } + } + + var lastItem *ini.Key + for i := first; i < len(itemTree); i++ { + keyVal := itemTree[i] + if comment, ok := keyVal.Key.(sops.Comment); ok { + if lastItem != nil { + lastItem.Comment = comment.Value + } + } else { + lastItem, err = section.NewKey(keyVal.Key.(string), store.valToString(keyVal.Value)) + if err != nil { + return nil, fmt.Errorf("Error encoding key: %s", err) + } + } + } + } + } + var buffer bytes.Buffer + iniFile.WriteTo(&buffer) + return buffer.Bytes(), nil +} + +func (store Store) stripCommentChar(comment string) string { + if strings.HasPrefix(comment, ";") { + comment = strings.TrimLeft(comment, "; ") + } else if strings.HasPrefix(comment, "#") { + comment = strings.TrimLeft(comment, "# ") + } + return comment +} + +func (store Store) valToString(v interface{}) string { + switch v := v.(type) { + case fmt.Stringer: + return v.String() + case float64: + return strconv.FormatFloat(v, 'f', 6, 64) + case bool: + return strconv.FormatBool(v) + default: + return fmt.Sprintf("%s", v) + } +} + +func (store Store) iniFromTreeBranches(branches sops.TreeBranches) ([]byte, error) { + return store.encodeTree(branches) +} + +func (store Store) treeBranchesFromIni(in []byte) (sops.TreeBranches, error) { + iniFile, err := ini.Load(in) + if err != nil { + return nil, err + } + var branch sops.TreeBranch + for _, section := range iniFile.Sections() { + + item, err := store.treeItemFromSection(section) + if err != nil { + return sops.TreeBranches{branch}, err + } + branch = append(branch, item) + } + return sops.TreeBranches{branch}, nil +} + +func (store Store) treeItemFromSection(section *ini.Section) (sops.TreeItem, error) { + var sectionItem sops.TreeItem + sectionItem.Key = section.Name() + var items sops.TreeBranch + + if section.Comment != "" { + items = append(items, sops.TreeItem{ + Key: sops.Comment{ + Value: store.stripCommentChar(section.Comment), + }, + Value: nil, + }) + } + + for _, key := range section.Keys() { + item := sops.TreeItem{Key:key.Name(), Value:key.Value()} + items = append(items, item) + if key.Comment != "" { + items = append(items, sops.TreeItem{ + Key: sops.Comment{ + Value: store.stripCommentChar(key.Comment), + }, + Value: nil, + }) + } + } + sectionItem.Value = items + return sectionItem, nil +} + + +func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { + iniFileOuter, err := ini.Load(in) + if err != nil { + return sops.Tree{}, err + } + + sopsSection, err := iniFileOuter.GetSection("sops") + if err != nil { + return sops.Tree{}, sops.MetadataNotFound + } + + metadataHolder, err := store.iniSectionToMetadata(sopsSection) + if err != nil { + return sops.Tree{}, err + } + + metadata, err := metadataHolder.ToInternal() + if err != nil { + return sops.Tree{}, err + } + // After that, we load the whole file into a map. + branches, err := store.treeBranchesFromIni(in) + if err != nil { + return sops.Tree{}, fmt.Errorf("Could not unmarshal input data: %s", err) + } + // Discard metadata, as we already loaded it. + for bi, branch := range branches { + for s, sectionBranch := range branch { + if sectionBranch.Key == "sops"{ + branch = append(branch[:s], branch[s+1:]...) + branches[bi] = branch + } + } + } + return sops.Tree{ + Branches: branches, + Metadata: metadata, + }, nil +} + +func (store *Store) iniSectionToMetadata(sopsSection *ini.Section) (stores.Metadata, error) { + + metadata := stores.Metadata{} + m := reflect.ValueOf(&metadata).Elem() + + for _, key := range sopsSection.Keys() { + + if strings.Contains(key.Name(), ".") { + parts := strings.SplitN(key.Name(), ".", 2) + if len(parts) != 2 { + return metadata, fmt.Errorf("Bad metadata format: key %s makes no sense", key.Name()) + } + prefix := parts[0] + remainder := parts[1] + // Is a slice + if strings.Contains(prefix, "[") { + k, i, err := parseSliceKey(prefix) + if err != nil { + return metadata, fmt.Errorf("Bad metadata format: %s", err) + } + + f := m.FieldByName(k) + ensureReflectedSliceLength(f, i+1) + sliceItem := f.Index(i) + setMetadataField(sliceItem, remainder, key) + } else { + return metadata, fmt.Errorf("Bad metadata format: expected array but have %s", prefix) + } + } else { + err := setMetadataField(m, key.Name(), key) + if err != nil { + return metadata, err + } + } + } + + return metadata, nil +} + +func ensureReflectedSliceLength(slice reflect.Value, length int) { + if slice.Len() < length { + expanded := reflect.MakeSlice(slice.Type(), slice.Len()+1, slice.Cap()+1) + reflect.Copy(expanded, slice) + slice.Set(expanded) + } +} + +func setMetadataField(m reflect.Value, name string, iniKey *ini.Key) error { + f := m.FieldByName(name) + switch f.Kind() { + case reflect.String: + f.SetString(strings.Replace(iniKey.String(), "\\n", "\n", -1)) + case reflect.Int: + val, err := iniKey.Int64() + if err != nil { + return err + } + f.SetInt(val) + } + return nil +} + +func parseSliceKey(key string) (string, int, error) { + openBracket := strings.IndexRune(key, '[') + closeBracket := strings.IndexRune(key, ']') + name := key[:openBracket] + indexStr := key[openBracket+1:closeBracket] + i, err := strconv.Atoi(indexStr) + return name, i, err +} + +func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { + branches, err := store.treeBranchesFromIni(in) + if err != nil { + return branches, fmt.Errorf("Could not unmarshal input data: %s", err) + } + return branches, nil +} + +func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { + + metadata := stores.MetadataFromInternal(in.Metadata) + newBranch, err := store.encodeMetadataToIniBranch(metadata, "sops") + if err != nil { + return nil, err + } + sectionItem := sops.TreeItem{Key: "sops", Value: newBranch} + branch := sops.TreeBranch{sectionItem} + + in.Branches = append(in.Branches, branch) + + out, err := store.iniFromTreeBranches(in.Branches) + if err != nil { + return nil, fmt.Errorf("Error marshaling to ini: %s", err) + } + return out, nil +} + +func (store *Store) encodeMetadataToIniBranch(metadata interface{}, prefix string) (sops.TreeBranch, error) { + + branch := sops.TreeBranch{} + + m := reflect.ValueOf(metadata) + r, err := encodeMetadataItem("", m.Type().Kind(), m) + + // Keys are sorted so sops section is stable (for nice diffs) + var keys []string + for k := range r { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + branch = append(branch, sops.TreeItem{Key: k, Value: r[k]}) + } + + return branch, err +} + +func encodeMetadataItem(prefix string, kind reflect.Kind, field reflect.Value) (map[string]interface{}, error) { + + result := make(map[string]interface{}, 0) + + switch kind { + case reflect.Slice: + slf := field + for j := 0; j < slf.Len(); j++ { + item := slf.Index(j) + p := fmt.Sprintf("%s[%d]", prefix, j) + r, err := encodeMetadataItem(p, item.Type().Kind(), item) + if err != nil { + return result, err + } + for k, v := range r { + result[k] = v + } + } + case reflect.Struct: + for i := 0; i < field.NumField(); i++ { + sf := field.Type().Field(i) + var name string + if prefix == "" { + name = sf.Name + } else { + name = fmt.Sprintf("%s.%s", prefix, sf.Name) + } + r, err := encodeMetadataItem(name, sf.Type.Kind(), field.Field(i)) + if err != nil { + return result, err + } + for k, v := range r { + result[k] = v + } + } + case reflect.Int: + if field.Int() != 0 { + result[prefix] = string(field.Int()) + } + case reflect.String: + if field.String() != "" { + result[prefix] = strings.Replace(field.String(), "\n", "\\n", -1) + } + default: + return result, fmt.Errorf("Cannot encode %s, unexpected type %s", prefix, kind) + } + + return result, nil +} + + +func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { + out, err := store.iniFromTreeBranches(in) + if err != nil { + return nil, fmt.Errorf("Error marshaling to ini: %s", err) + } + return out, nil +} + +func (store Store) encodeValue(v interface{}) ([]byte, error) { + switch v := v.(type) { + case sops.TreeBranches: + return store.encodeTree(v) + default: + return json.Marshal(v) + } +} + +func (store *Store) EmitValue(v interface{}) ([]byte, error) { + return store.encodeValue(v) +} diff --git a/stores/ini/store_test.go b/stores/ini/store_test.go new file mode 100644 index 000000000..58b28e03a --- /dev/null +++ b/stores/ini/store_test.go @@ -0,0 +1,136 @@ +package ini + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.mozilla.org/sops" +) + +func TestDecodeIni(t *testing.T) { + in := ` +; last modified 1 April 2001 by John Doe +[owner] +name=John Doe +organization=Acme Widgets Inc. + +[database] +; use IP address in case network name resolution is not working +server=192.0.2.62 +port=143 +file="payroll.dat" +` + expected := sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "DEFAULT", + Value: sops.TreeBranch(nil), + }, + sops.TreeItem{ + Key: "owner", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: sops.Comment{Value: "last modified 1 April 2001 by John Doe"}, + Value: nil, + }, + sops.TreeItem{ + Key: "name", + Value: "John Doe", + }, + sops.TreeItem{ + Key: "organization", + Value: "Acme Widgets Inc.", + }, + }, + }, + sops.TreeItem{ + Key: "database", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "server", + Value: "192.0.2.62", + }, + sops.TreeItem{ + Key: sops.Comment{Value: "use IP address in case network name resolution is not working"}, + Value: nil, + }, + sops.TreeItem{ + Key: "port", + Value: "143", + }, + sops.TreeItem{ + Key: "file", + Value: "payroll.dat", + }, + }, + }, + }, + } + branch, err := Store{}.treeBranchesFromIni([]byte(in)) + assert.Nil(t, err) + assert.Equal(t, expected, branch) +} + +func TestEncodeSimpleIni(t *testing.T) { + branches := sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "DEFAULT", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: "bar", + }, + sops.TreeItem{ + Key: "baz", + Value: "3.0", + }, + sops.TreeItem{ + Key: "qux", + Value: "false", + }, + }, + }, + }, + } + out, err := Store{}.iniFromTreeBranches(branches) + assert.Nil(t, err) + expected, _ := Store{}.treeBranchesFromIni(out) + assert.Equal(t, expected, branches) +} + +func TestEncodeIniWithEscaping(t *testing.T) { + branches := sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "DEFAULT", + Value: sops.TreeBranch{ + sops.TreeItem{ + Key: "foo\\bar", + Value: "value", + }, + sops.TreeItem{ + Key: "a_key_with\"quotes\"", + Value: "4.0", + }, + sops.TreeItem{ + Key: "baz\\\\foo", + Value: "2.0", + }, + }, + }, + }, + } + out, err := Store{}.iniFromTreeBranches(branches) + assert.Nil(t, err) + expected, _ := Store{}.treeBranchesFromIni(out) + assert.Equal(t, expected, branches) +} + +func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { + data := []byte(`hello=2`) + store := Store{} + _, err := store.LoadEncryptedFile(data) + assert.Equal(t, sops.MetadataNotFound, err) +} + diff --git a/stores/ini/test_resources/example.json b/stores/ini/test_resources/example.json new file mode 100644 index 000000000..594f99d42 --- /dev/null +++ b/stores/ini/test_resources/example.json @@ -0,0 +1,33 @@ +{ + "example_key": "ENC[AES256_GCM,data:Xjen3YMQYCBfTU8VjA==,iv:1NUKversqeQiuTmAkZuyd6UY2AWiBS4owa4QHnwKOBM=,tag:ZO+Aln5DQ5Qm/QV2uBdahA==,type:str]", + "example_array": [ + "ENC[AES256_GCM,data:XJR0qvZuifm8j1TIQB8=,iv:QUkwy1dp0RU0PKEAw/VxVe1ZsQ972c8gPMJoVKgMfuw=,tag:LGdwC5nTua4rSe0dLbbA1Q==,type:str]", + "ENC[AES256_GCM,data:7bhghzi5GN/mMqh1vHU=,iv:X5vrd9X7ItIG/RVCn0T7RFhUrTb2YItr3i97EVk9nOY=,tag:vrM058PPOGmWOGPThOP5Ew==,type:str]" + ], + "example_number": "ENC[AES256_GCM,data:w9etQN5r8iCz,iv:YF+1uUlMa4I1C7A0ELpVuMa1yK042uEMhp8y6HiCTDE=,tag:Dmh+AV9sh+ir0M1Txe+v2A==,type:float]", + "example_booleans": [ + "ENC[AES256_GCM,data:/hltsg==,iv:pbAtZ9i8rxFpaFlbwE1KOA+k/TVx5dm0tDtH94GCEVc=,tag:yz3d1pQu9zy5Ra9z2kDcoA==,type:bool]", + "ENC[AES256_GCM,data:HEei0+s=,iv:hgKT5eiYdHn5AqWdNji7vRKfabln90VbLmJqt7A480E=,tag:6pfezzTX/eULebF+32Z2+w==,type:bool]" + ], + "sops": { + "lastmodified": "2016-08-04T23:30:35Z", + "attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", + "unencrypted_suffix": "_unencrypted", + "mac": "ENC[AES256_GCM,data:EK1LkVgW5CBEsGgGc7RkfZlzqWrP2fZe3kG7HbkJ5JFd591oUkbQ6I2uPImkcxf7HjiEHzKPPF5QvNg3+rUxgw6S8pQtumhDbFrfDi8GDS2VVvPR+0fnc2fR5PMGm36bOaQFDNSmgyJzKhMmNL+MtRhH+fMUnHhrnxuN3wfLr4w=,iv:xbNK6wRDVT4xhrP+vP2RIy+uNjZSSzqEJZPOdShn96o=,tag:vT4akR5X6qx5/wJ4dncxtg==,type:str]", + "version": "1.13", + "kms": [ + { + "created_at": "2016-08-04T23:30:35Z", + "enc": "AQECAHgFEiO2dNygC3Rz8PhERCc8Sfhak4g81FUPqQJ0OBcAKgAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDPKe5R67LMN3+xAkygIBEIA7F8noZukawV3VLQ/yH3Ep7Ptx8weLFUgVf/ZI6xqSMNvEHIr4+vf2xjBiAyrEF8u/n9nm9PWAdKHszFM=", + "arn": "arn:aws:kms:us-east-1:927034868273:key/e9fc75db-05e9-44c1-9c35-633922bac347" + } + ], + "pgp": [ + { + "fp": "E5297818703249D0C60E19E6824612478D1A4CCD", + "created_at": "2016-08-04T23:30:35Z", + "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA2X8rvoeiASBARAAurTEVS82kqadk68f5ZlwR176S148WYTYxFp5oMC7cVD7\n42+Eo9RzaxbHeO5n7XKX0SDOUUeCucFl8fwuDUV1iDIx4/u5HgWXxDuvWoNe5cAL\n4LBS1Er2ZBVAdU0WHZ/8USuZLhSu7ucAHOvqNpHzPT6gkuBUYLQKOu0c+onWHqVO\n1DhfkTtvphotZ0ZBBR099t5N8ofD0W2+SM268A9/bB5yQcK9Ig/KxZBrfmMQm7zx\n9hLVQhcBmj0OQG37K4/SXGwjrQFarh6lm+FuZM0Q+GI+OARoAKdpZOnPEhXKE6un\nSEa69rh5FKVM/XRp2/QVZEakzRtq3gi9CtYL2sNr7KEnCvxt/v2pEc6evfIvxWTc\nT8MWdk48FkVjdsJ34sNiIM8msstnYorse8RZny9gcLE+A5lsRavo2QPL4GADyHF8\n7kwijSVDd08nByTBMMEPpMozUFhzF8QuVZPD+siuUvi+Bned9MmqgGMfvhS0Kf38\nMZFy5C6e38VGEX3IrWChvzbBm/M3fjs1fPVDShHfk1MYsCU9sXNQMQVewWE0s/em\nklycIL3hywd4N9z1MVW2hBpRrC247PtGQRKGoB9qbKtSgjTtgM7bo1vYekeY1tjr\nBGTHNFV+FBqFih16u/rGVzIaBsf5lLL/RtpaFZx1OWHMd9XjQpRrHhjOMpQ8tjvS\nXgGH59vv/9GNZ+Rix1QF+iMD84sfkyyguGKwg+TC3m275v+HIO1NvNdU6oS3O/Xq\nBCBV3yYAUwcrWUPWCuSUHJbuHKJEI1ymXUu8+RUElPyi/5JEhW+J1WlVPvnG1Xk=\n=Q/XA\n-----END PGP MESSAGE-----\n" + } + ] + } +} \ No newline at end of file diff --git a/stores/stores.go b/stores/stores.go index 19d860a4b..7693572c2 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -27,7 +27,7 @@ type SopsFile struct { // in the SOPS file by checking for nil. This way we can show the user a // helpful error message indicating that the metadata wasn't found, instead // of showing a cryptic parsing error - Metadata *Metadata `yaml:"sops" json:"sops"` + Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops"` } // Metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file.