Added support for ini files

2026-02-05 12:45:21 +01:00 · 2018-08-26 00:18:10 -07:00
parent 88917267e0
commit c70c52be58
6 changed files with 551 additions and 1 deletions
--- a/cmd/sops/common/common.go
+++ b/cmd/sops/common/common.go
@@ -13,6 +13,7 @@ import (
 	"go.mozilla.org/sops/stores/json"
 	"go.mozilla.org/sops/stores/yaml"
 	"go.mozilla.org/sops/stores/dotenv"
+	"go.mozilla.org/sops/stores/ini"
 	"gopkg.in/urfave/cli.v1"
 )

@@ -109,6 +110,10 @@ func IsEnvFile(path string) bool {
 	return strings.HasSuffix(path, ".env")
 }

+func IsIniFile(path string) bool {
+	return strings.HasSuffix(path, ".ini")
+}
+
 func DefaultStoreForPath(path string) sops.Store {
 	if IsYAMLFile(path) {
 		return &yaml.Store{}
@@ -116,6 +121,8 @@ func DefaultStoreForPath(path string) sops.Store {
 		return &json.Store{}
 	} else if IsEnvFile(path) {
 		return &dotenv.Store{}
+	} else if IsIniFile(path) {
+		return &ini.Store{}
 	}
 	return &json.BinaryStore{}
 }
--- a/cmd/sops/main.go
+++ b/cmd/sops/main.go
@@ -33,6 +33,7 @@ import (
 	yamlstores "go.mozilla.org/sops/stores/yaml"
 	"google.golang.org/grpc"
 	"gopkg.in/urfave/cli.v1"
+	"go.mozilla.org/sops/stores/ini"
 )

 var log *logrus.Logger
@@ -690,6 +691,8 @@ func inputStore(context *cli.Context, path string) sops.Store {
 		return &json.Store{}
 	case "dotenv":
 		return &dotenv.Store{}
+	case "ini":
+		return &ini.Store{}
 	case "binary":
 		return &json.BinaryStore{}
 	default:
@@ -705,6 +708,8 @@ func outputStore(context *cli.Context, path string) sops.Store {
 		return &json.Store{}
 	case "dotenv":
 		return &dotenv.Store{}
+	case "ini":
+		return &ini.Store{}
 	case "binary":
 		return &json.BinaryStore{}
 	default:
--- a/stores/ini/store.go
+++ b/stores/ini/store.go
@@ -0,0 +1,369 @@
+package ini //import "go.mozilla.org/sops/stores/ini"
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"go.mozilla.org/sops"
+	"go.mozilla.org/sops/stores"
+	"strconv"
+	"reflect"
+	"gopkg.in/ini.v1"
+	"strings"
+	"sort"
+)
+
+// Store handles storage of ini data.
+type Store struct {
+}
+
+func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) {
+	iniFile := ini.Empty()
+	for _, branch := range branches {
+		for _, item := range branch {
+			if _, ok := item.Key.(sops.Comment); ok {
+				continue
+			}
+			section, err := iniFile.NewSection(item.Key.(string))
+			if err != nil {
+				return nil, fmt.Errorf("Error encoding section %s: %s", item.Key, err)
+			}
+			itemTree, ok := item.Value.(sops.TreeBranch)
+			if !ok {
+				return nil, fmt.Errorf("Error encoding section: Section values should always be TreeBranches")
+			}
+
+			first := 0
+			if len(itemTree) > 0 {
+				if sectionComment, ok := itemTree[0].Key.(sops.Comment); ok {
+					section.Comment = sectionComment.Value
+					first = 1
+				}
+			}
+
+			var lastItem *ini.Key
+			for i := first; i < len(itemTree); i++ {
+				keyVal := itemTree[i]
+				if comment, ok := keyVal.Key.(sops.Comment); ok {
+					if lastItem != nil {
+						lastItem.Comment = comment.Value
+					}
+				} else {
+					lastItem, err = section.NewKey(keyVal.Key.(string), store.valToString(keyVal.Value))
+					if err != nil {
+						return nil, fmt.Errorf("Error encoding key: %s", err)
+					}
+				}
+			}
+		}
+	}
+	var buffer bytes.Buffer
+	iniFile.WriteTo(&buffer)
+	return buffer.Bytes(), nil
+}
+
+func (store Store) stripCommentChar(comment string) string {
+	if strings.HasPrefix(comment, ";") {
+		comment = strings.TrimLeft(comment, "; ")
+	} else if strings.HasPrefix(comment, "#") {
+		comment = strings.TrimLeft(comment, "# ")
+	}
+	return comment
+}
+
+func (store Store) valToString(v interface{}) string {
+	switch v := v.(type) {
+	case fmt.Stringer:
+		return v.String()
+	case float64:
+		return strconv.FormatFloat(v, 'f', 6, 64)
+	case bool:
+		return strconv.FormatBool(v)
+	default:
+		return fmt.Sprintf("%s", v)
+	}
+}
+
+func (store Store) iniFromTreeBranches(branches sops.TreeBranches) ([]byte, error) {
+	return store.encodeTree(branches)
+}
+
+func (store Store) treeBranchesFromIni(in []byte) (sops.TreeBranches, error) {
+	iniFile, err := ini.Load(in)
+	if err != nil {
+		return nil, err
+	}
+	var branch sops.TreeBranch
+	for _, section := range iniFile.Sections() {
+
+		item, err := store.treeItemFromSection(section)
+		if err != nil {
+			return sops.TreeBranches{branch}, err
+		}
+		branch = append(branch, item)
+	}
+	return sops.TreeBranches{branch}, nil
+}
+
+func (store Store) treeItemFromSection(section *ini.Section) (sops.TreeItem, error) {
+	var sectionItem sops.TreeItem
+	sectionItem.Key = section.Name()
+	var items sops.TreeBranch
+
+	if section.Comment != "" {
+		items = append(items, sops.TreeItem{
+			Key: sops.Comment{
+				Value: store.stripCommentChar(section.Comment),
+			},
+			Value: nil,
+		})
+	}
+
+	for _, key := range section.Keys() {
+		item := sops.TreeItem{Key:key.Name(), Value:key.Value()}
+		items = append(items, item)
+		if key.Comment != "" {
+			items = append(items, sops.TreeItem{
+				Key: sops.Comment{
+					Value: store.stripCommentChar(key.Comment),
+				},
+				Value: nil,
+			})
+		}
+	}
+	sectionItem.Value = items
+	return sectionItem, nil
+}
+
+
+func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) {
+	iniFileOuter, err := ini.Load(in)
+	if err != nil {
+		return sops.Tree{}, err
+	}
+
+	sopsSection, err := iniFileOuter.GetSection("sops")
+	if err != nil {
+		return sops.Tree{}, sops.MetadataNotFound
+	}
+
+	metadataHolder, err := store.iniSectionToMetadata(sopsSection)
+	if err != nil {
+		return sops.Tree{}, err
+	}
+
+	metadata, err := metadataHolder.ToInternal()
+	if err != nil {
+		return sops.Tree{}, err
+	}
+	// After that, we load the whole file into a map.
+	branches, err := store.treeBranchesFromIni(in)
+	if err != nil {
+		return sops.Tree{}, fmt.Errorf("Could not unmarshal input data: %s", err)
+	}
+	// Discard metadata, as we already loaded it.
+	for bi, branch := range branches {
+		for s, sectionBranch := range branch {
+			if sectionBranch.Key == "sops"{
+				branch = append(branch[:s], branch[s+1:]...)
+				branches[bi] = branch
+			}
+		}
+	}
+	return sops.Tree{
+		Branches: branches,
+		Metadata: metadata,
+	}, nil
+}
+
+func (store *Store) iniSectionToMetadata(sopsSection *ini.Section) (stores.Metadata, error) {
+
+	metadata := stores.Metadata{}
+	m := reflect.ValueOf(&metadata).Elem()
+
+	for _, key := range sopsSection.Keys() {
+
+		if strings.Contains(key.Name(), ".") {
+			parts := strings.SplitN(key.Name(), ".", 2)
+			if len(parts) != 2 {
+				return metadata, fmt.Errorf("Bad metadata format: key %s makes no sense", key.Name())
+			}
+			prefix := parts[0]
+			remainder := parts[1]
+			// Is a slice
+			if strings.Contains(prefix, "[") {
+				k, i, err := parseSliceKey(prefix)
+				if err != nil {
+					return metadata, fmt.Errorf("Bad metadata format: %s", err)
+				}
+
+				f := m.FieldByName(k)
+				ensureReflectedSliceLength(f, i+1)
+				sliceItem := f.Index(i)
+				setMetadataField(sliceItem, remainder, key)
+			} else {
+				return metadata, fmt.Errorf("Bad metadata format: expected array but have %s", prefix)
+			}
+		} else {
+			err := setMetadataField(m, key.Name(), key)
+			if err != nil {
+				return metadata, err
+			}
+		}
+	}
+
+	return metadata, nil
+}
+
+func ensureReflectedSliceLength(slice reflect.Value, length int) {
+	if slice.Len() < length  {
+		expanded := reflect.MakeSlice(slice.Type(), slice.Len()+1, slice.Cap()+1)
+		reflect.Copy(expanded, slice)
+		slice.Set(expanded)
+	}
+}
+
+func setMetadataField(m reflect.Value, name string, iniKey *ini.Key) error {
+	f := m.FieldByName(name)
+	switch f.Kind() {
+	case reflect.String:
+		f.SetString(strings.Replace(iniKey.String(), "\\n", "\n", -1))
+	case reflect.Int:
+		val, err := iniKey.Int64()
+		if err != nil {
+			return err
+		}
+		f.SetInt(val)
+	}
+	return nil
+}
+
+func parseSliceKey(key string) (string, int, error) {
+	openBracket := strings.IndexRune(key, '[')
+	closeBracket := strings.IndexRune(key, ']')
+	name := key[:openBracket]
+	indexStr := key[openBracket+1:closeBracket]
+	i, err := strconv.Atoi(indexStr)
+	return name, i, err
+}
+
+func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) {
+	branches, err := store.treeBranchesFromIni(in)
+	if err != nil {
+		return branches, fmt.Errorf("Could not unmarshal input data: %s", err)
+	}
+	return branches, nil
+}
+
+func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) {
+
+	metadata := stores.MetadataFromInternal(in.Metadata)
+	newBranch, err := store.encodeMetadataToIniBranch(metadata, "sops")
+	if err != nil {
+		return nil, err
+	}
+	sectionItem := sops.TreeItem{Key: "sops", Value: newBranch}
+	branch := sops.TreeBranch{sectionItem}
+
+	in.Branches = append(in.Branches, branch)
+
+	out, err := store.iniFromTreeBranches(in.Branches)
+	if err != nil {
+		return nil, fmt.Errorf("Error marshaling to ini: %s", err)
+	}
+	return out, nil
+}
+
+func (store *Store) encodeMetadataToIniBranch(metadata interface{}, prefix string) (sops.TreeBranch, error) {
+
+	branch := sops.TreeBranch{}
+
+	m := reflect.ValueOf(metadata)
+	r, err := encodeMetadataItem("", m.Type().Kind(), m)
+
+	// Keys are sorted so sops section is stable (for nice diffs)
+	var keys []string
+	for k := range r {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		branch = append(branch, sops.TreeItem{Key: k, Value: r[k]})
+	}
+
+	return branch, err
+}
+
+func encodeMetadataItem(prefix string, kind reflect.Kind, field reflect.Value) (map[string]interface{}, error) {
+
+	result := make(map[string]interface{}, 0)
+
+	switch kind {
+	case reflect.Slice:
+		slf := field
+		for j := 0; j < slf.Len(); j++ {
+			item := slf.Index(j)
+			p := fmt.Sprintf("%s[%d]", prefix, j)
+			r, err := encodeMetadataItem(p, item.Type().Kind(), item)
+			if err != nil {
+				return result, err
+			}
+			for k, v := range r {
+				result[k] = v
+			}
+		}
+	case reflect.Struct:
+		for i := 0; i < field.NumField(); i++ {
+			sf := field.Type().Field(i)
+			var name string
+			if prefix == "" {
+				name = sf.Name
+			} else {
+				name = fmt.Sprintf("%s.%s", prefix, sf.Name)
+			}
+			r, err := encodeMetadataItem(name, sf.Type.Kind(), field.Field(i))
+			if err != nil {
+				return result, err
+			}
+			for k, v := range r {
+				result[k] = v
+			}
+		}
+	case reflect.Int:
+		if field.Int() != 0 {
+			result[prefix] = string(field.Int())
+		}
+	case reflect.String:
+		if field.String() != "" {
+			result[prefix] = strings.Replace(field.String(), "\n", "\\n", -1)
+		}
+	default:
+		return result, fmt.Errorf("Cannot encode %s, unexpected type %s", prefix, kind)
+	}
+
+	return result, nil
+}
+
+
+func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) {
+	out, err := store.iniFromTreeBranches(in)
+	if err != nil {
+		return nil, fmt.Errorf("Error marshaling to ini: %s", err)
+	}
+	return out, nil
+}
+
+func (store Store) encodeValue(v interface{}) ([]byte, error) {
+	switch v := v.(type) {
+	case sops.TreeBranches:
+		return store.encodeTree(v)
+	default:
+		return json.Marshal(v)
+	}
+}
+
+func (store *Store) EmitValue(v interface{}) ([]byte, error) {
+	return store.encodeValue(v)
+}
--- a/stores/ini/store_test.go
+++ b/stores/ini/store_test.go
@@ -0,0 +1,136 @@
+package ini
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.mozilla.org/sops"
+)
+
+func TestDecodeIni(t *testing.T) {
+	in := `
+; last modified 1 April 2001 by John Doe
+[owner]
+name=John Doe
+organization=Acme Widgets Inc.
+
+[database]
+; use IP address in case network name resolution is not working
+server=192.0.2.62     
+port=143
+file="payroll.dat"
+`
+	expected := sops.TreeBranches{
+		sops.TreeBranch{
+			sops.TreeItem{
+				Key:   "DEFAULT",
+				Value: sops.TreeBranch(nil),
+			},
+			sops.TreeItem{
+				Key: "owner",
+				Value: sops.TreeBranch{
+					sops.TreeItem{
+						Key:   sops.Comment{Value: "last modified 1 April 2001 by John Doe"},
+						Value: nil,
+					},
+					sops.TreeItem{
+						Key:   "name",
+						Value: "John Doe",
+					},
+					sops.TreeItem{
+						Key:   "organization",
+						Value: "Acme Widgets Inc.",
+					},
+				},
+			},
+			sops.TreeItem{
+				Key: "database",
+				Value: sops.TreeBranch{
+					sops.TreeItem{
+						Key:   "server",
+						Value: "192.0.2.62",
+					},
+					sops.TreeItem{
+						Key:   sops.Comment{Value: "use IP address in case network name resolution is not working"},
+						Value: nil,
+					},
+					sops.TreeItem{
+						Key:   "port",
+						Value: "143",
+					},
+					sops.TreeItem{
+						Key:   "file",
+						Value: "payroll.dat",
+					},
+				},
+			},
+		},
+	}
+	branch, err := Store{}.treeBranchesFromIni([]byte(in))
+	assert.Nil(t, err)
+	assert.Equal(t, expected, branch)
+}
+
+func TestEncodeSimpleIni(t *testing.T) {
+	branches := sops.TreeBranches{
+		sops.TreeBranch{
+			sops.TreeItem{
+				Key: "DEFAULT",
+				Value: sops.TreeBranch{
+					sops.TreeItem{
+						Key:   "foo",
+						Value: "bar",
+					},
+					sops.TreeItem{
+						Key:   "baz",
+						Value: "3.0",
+					},
+					sops.TreeItem{
+						Key:   "qux",
+						Value: "false",
+					},
+				},
+			},
+		},
+	}
+	out, err := Store{}.iniFromTreeBranches(branches)
+	assert.Nil(t, err)
+	expected, _ := Store{}.treeBranchesFromIni(out)
+	assert.Equal(t, expected, branches)
+}
+
+func TestEncodeIniWithEscaping(t *testing.T) {
+	branches := sops.TreeBranches{
+		sops.TreeBranch{
+			sops.TreeItem{
+				Key: "DEFAULT",
+				Value: sops.TreeBranch{
+					sops.TreeItem{
+						Key:   "foo\\bar",
+						Value: "value",
+					},
+					sops.TreeItem{
+						Key:   "a_key_with\"quotes\"",
+						Value: "4.0",
+					},
+					sops.TreeItem{
+						Key:   "baz\\\\foo",
+						Value: "2.0",
+					},
+				},
+			},
+		},
+	}
+	out, err := Store{}.iniFromTreeBranches(branches)
+	assert.Nil(t, err)
+	expected, _ := Store{}.treeBranchesFromIni(out)
+	assert.Equal(t, expected, branches)
+}
+
+func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) {
+	data := []byte(`hello=2`)
+	store := Store{}
+	_, err := store.LoadEncryptedFile(data)
+	assert.Equal(t, sops.MetadataNotFound, err)
+}
+
--- a/stores/ini/test_resources/example.json
+++ b/stores/ini/test_resources/example.json
@@ -0,0 +1,33 @@
+{
+    "example_key": "ENC[AES256_GCM,data:Xjen3YMQYCBfTU8VjA==,iv:1NUKversqeQiuTmAkZuyd6UY2AWiBS4owa4QHnwKOBM=,tag:ZO+Aln5DQ5Qm/QV2uBdahA==,type:str]", 
+    "example_array": [
+        "ENC[AES256_GCM,data:XJR0qvZuifm8j1TIQB8=,iv:QUkwy1dp0RU0PKEAw/VxVe1ZsQ972c8gPMJoVKgMfuw=,tag:LGdwC5nTua4rSe0dLbbA1Q==,type:str]", 
+        "ENC[AES256_GCM,data:7bhghzi5GN/mMqh1vHU=,iv:X5vrd9X7ItIG/RVCn0T7RFhUrTb2YItr3i97EVk9nOY=,tag:vrM058PPOGmWOGPThOP5Ew==,type:str]"
+    ], 
+    "example_number": "ENC[AES256_GCM,data:w9etQN5r8iCz,iv:YF+1uUlMa4I1C7A0ELpVuMa1yK042uEMhp8y6HiCTDE=,tag:Dmh+AV9sh+ir0M1Txe+v2A==,type:float]", 
+    "example_booleans": [
+        "ENC[AES256_GCM,data:/hltsg==,iv:pbAtZ9i8rxFpaFlbwE1KOA+k/TVx5dm0tDtH94GCEVc=,tag:yz3d1pQu9zy5Ra9z2kDcoA==,type:bool]", 
+        "ENC[AES256_GCM,data:HEei0+s=,iv:hgKT5eiYdHn5AqWdNji7vRKfabln90VbLmJqt7A480E=,tag:6pfezzTX/eULebF+32Z2+w==,type:bool]"
+    ], 
+    "sops": {
+        "lastmodified": "2016-08-04T23:30:35Z", 
+        "attention": "This section contains key material that should only be modified with extra care. See `sops -h`.", 
+        "unencrypted_suffix": "_unencrypted", 
+        "mac": "ENC[AES256_GCM,data:EK1LkVgW5CBEsGgGc7RkfZlzqWrP2fZe3kG7HbkJ5JFd591oUkbQ6I2uPImkcxf7HjiEHzKPPF5QvNg3+rUxgw6S8pQtumhDbFrfDi8GDS2VVvPR+0fnc2fR5PMGm36bOaQFDNSmgyJzKhMmNL+MtRhH+fMUnHhrnxuN3wfLr4w=,iv:xbNK6wRDVT4xhrP+vP2RIy+uNjZSSzqEJZPOdShn96o=,tag:vT4akR5X6qx5/wJ4dncxtg==,type:str]", 
+        "version": "1.13", 
+        "kms": [
+            {
+                "created_at": "2016-08-04T23:30:35Z", 
+                "enc": "AQECAHgFEiO2dNygC3Rz8PhERCc8Sfhak4g81FUPqQJ0OBcAKgAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDPKe5R67LMN3+xAkygIBEIA7F8noZukawV3VLQ/yH3Ep7Ptx8weLFUgVf/ZI6xqSMNvEHIr4+vf2xjBiAyrEF8u/n9nm9PWAdKHszFM=", 
+                "arn": "arn:aws:kms:us-east-1:927034868273:key/e9fc75db-05e9-44c1-9c35-633922bac347"
+            }
+        ], 
+        "pgp": [
+            {
+                "fp": "E5297818703249D0C60E19E6824612478D1A4CCD", 
+                "created_at": "2016-08-04T23:30:35Z", 
+                "enc": "-----BEGIN PGP MESSAGE-----\nVersion: GnuPG v1\n\nhQIMA2X8rvoeiASBARAAurTEVS82kqadk68f5ZlwR176S148WYTYxFp5oMC7cVD7\n42+Eo9RzaxbHeO5n7XKX0SDOUUeCucFl8fwuDUV1iDIx4/u5HgWXxDuvWoNe5cAL\n4LBS1Er2ZBVAdU0WHZ/8USuZLhSu7ucAHOvqNpHzPT6gkuBUYLQKOu0c+onWHqVO\n1DhfkTtvphotZ0ZBBR099t5N8ofD0W2+SM268A9/bB5yQcK9Ig/KxZBrfmMQm7zx\n9hLVQhcBmj0OQG37K4/SXGwjrQFarh6lm+FuZM0Q+GI+OARoAKdpZOnPEhXKE6un\nSEa69rh5FKVM/XRp2/QVZEakzRtq3gi9CtYL2sNr7KEnCvxt/v2pEc6evfIvxWTc\nT8MWdk48FkVjdsJ34sNiIM8msstnYorse8RZny9gcLE+A5lsRavo2QPL4GADyHF8\n7kwijSVDd08nByTBMMEPpMozUFhzF8QuVZPD+siuUvi+Bned9MmqgGMfvhS0Kf38\nMZFy5C6e38VGEX3IrWChvzbBm/M3fjs1fPVDShHfk1MYsCU9sXNQMQVewWE0s/em\nklycIL3hywd4N9z1MVW2hBpRrC247PtGQRKGoB9qbKtSgjTtgM7bo1vYekeY1tjr\nBGTHNFV+FBqFih16u/rGVzIaBsf5lLL/RtpaFZx1OWHMd9XjQpRrHhjOMpQ8tjvS\nXgGH59vv/9GNZ+Rix1QF+iMD84sfkyyguGKwg+TC3m275v+HIO1NvNdU6oS3O/Xq\nBCBV3yYAUwcrWUPWCuSUHJbuHKJEI1ymXUu8+RUElPyi/5JEhW+J1WlVPvnG1Xk=\n=Q/XA\n-----END PGP MESSAGE-----\n"
+            }
+        ]
+    }
+}
--- a/stores/stores.go
+++ b/stores/stores.go
@@ -27,7 +27,7 @@ type SopsFile struct {
 	// in the SOPS file by checking for nil. This way we can show the user a
 	// helpful error message indicating that the metadata wasn't found, instead
 	// of showing a cryptic parsing error
-	Metadata *Metadata `yaml:"sops" json:"sops"`
+	Metadata *Metadata `yaml:"sops" json:"sops" ini:"sops"`
 }

 // Metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file.