Refactor metadata marshalling

**IMPORTANT** This breaks compatibility of the file format in 1.x for
json files, due to the version being encoded as a number in json files.
The fix for this is easy, however. One can either use a previous version
of sops in the range [2.0.0, 2.0.9] to edit the file, or one can manually edit
the encrypted file and change the version from a number to a string

Previously we basically hand-converted the metadata struct into a map
which we then passed to the stores. Now, we convert the metadata struct
to a "serialization" struct, which the stores serialize

Makefile

@@ -28,8 +28,9 @@ test:
 	$(GO) test -coverprofile=coverage_tmp.txt -covermode=atomic $(PROJECT) && cat coverage_tmp.txt >> coverage.txt
 	$(GO) test $(PROJECT)/aes -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
 	$(GO) test $(PROJECT)/cmd/sops -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
-	$(GO) test $(PROJECT)/json -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
 	$(GO) test $(PROJECT)/yaml -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
+	$(GO) test $(PROJECT)/stores/yaml -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
+	$(GO) test $(PROJECT)/stores/json -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
 	gpg --import pgp/sops_functional_tests_key.asc 2>&1 1>/dev/null || exit 0
 	$(GO) test $(PROJECT)/pgp -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt
 	$(GO) test $(PROJECT)/kms -coverprofile=coverage_tmp.txt -covermode=atomic && cat coverage_tmp.txt >> coverage.txt

cmd/sops/main.go

@@ -27,11 +27,12 @@ import (
 	"github.com/google/shlex"
 
 	"go.mozilla.org/sops/aes"
-	"go.mozilla.org/sops/json"
 	"go.mozilla.org/sops/keys"
 	"go.mozilla.org/sops/keyservice"
 	"go.mozilla.org/sops/kms"
 	"go.mozilla.org/sops/pgp"
+	"go.mozilla.org/sops/stores/json"
+	yamlstores "go.mozilla.org/sops/stores/yaml"
 	"go.mozilla.org/sops/yaml"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -98,7 +99,7 @@ func loadEncryptedFile(c *cli.Context, store sops.Store, fileBytes []byte) (tree
 	}
 	return sops.Tree{
 		Branch:   branch,
-		Metadata: metadata,
+		Metadata: *metadata,
 	}, nil
 }
 
@@ -251,9 +252,12 @@ func main() {
 
 		var output []byte
 		var err error
+		var encryptedTree sops.Tree
+		var fileBytes []byte
 		if c.Bool("encrypt") {
 			fileBytes, err := ioutil.ReadFile(fileName)
 			if err != nil {
+				panic(err)
 				return err
 			}
 			plainTree, err := loadPlainFile(c, inputStore, fileName, fileBytes, svcs)
@@ -264,22 +268,22 @@ func main() {
 		}
 
 		if c.Bool("decrypt") {
-			fileBytes, err := ioutil.ReadFile(fileName)
+			fileBytes, err = ioutil.ReadFile(fileName)
 			if err != nil {
 				return err
 			}
-			encryptedTree, err := loadEncryptedFile(c, inputStore, fileBytes)
+			encryptedTree, err = loadEncryptedFile(c, inputStore, fileBytes)
 			if err != nil {
 				return err
 			}
 			output, err = decrypt(c, encryptedTree, outputStore, svcs)
 		}
 		if c.Bool("rotate") {
-			fileBytes, err := ioutil.ReadFile(fileName)
+			fileBytes, err = ioutil.ReadFile(fileName)
 			if err != nil {
 				return err
 			}
-			encryptedTree, err := loadEncryptedFile(c, inputStore, fileBytes)
+			encryptedTree, err = loadEncryptedFile(c, inputStore, fileBytes)
 			if err != nil {
 				return err
 			}
@@ -288,10 +292,7 @@ func main() {
 
 		isEditMode := !c.Bool("encrypt") && !c.Bool("decrypt") && !c.Bool("rotate")
 		if isEditMode {
-			fileBytes, err := ioutil.ReadFile(fileName)
-			if err != nil {
-				return err
-			}
+			fileBytes, _ := ioutil.ReadFile(fileName)
 			output, err = edit(c, fileName, fileBytes, svcs)
 		}
 
@@ -375,7 +376,7 @@ func runEditor(path string) error {
 func inputStore(context *cli.Context, path string) sops.Store {
 	switch context.String("input-type") {
 	case "yaml":
-		return &yaml.Store{}
+		return &yamlstores.Store{}
 	case "json":
 		return &json.Store{}
 	default:
@@ -385,7 +386,7 @@ func inputStore(context *cli.Context, path string) sops.Store {
 func outputStore(context *cli.Context, path string) sops.Store {
 	switch context.String("output-type") {
 	case "yaml":
-		return &yaml.Store{}
+		return &yamlstores.Store{}
 	case "json":
 		return &json.Store{}
 	default:
@@ -395,7 +396,7 @@ func outputStore(context *cli.Context, path string) sops.Store {
 
 func defaultStore(path string) sops.Store {
 	if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
-		return &yaml.Store{}
+		return &yamlstores.Store{}
 	} else if strings.HasSuffix(path, ".json") {
 		return &json.Store{}
 	}
@@ -763,7 +764,7 @@ func edit(c *cli.Context, file string, fileBytes []byte, svcs []keyservice.KeySe
 					bufio.NewReader(os.Stdin).ReadByte()
 					continue
 				}
-				tree.Metadata = metadata
+				tree.Metadata = *metadata
 			}
 			tree.Branch = newBranch
 			needVersionUpdated, err := AIsNewerThanB(version, tree.Metadata.Version)

decrypt/decrypt.go

@@ -7,8 +7,8 @@ import (
 
 	"go.mozilla.org/sops"
 	"go.mozilla.org/sops/aes"
-	sopsjson "go.mozilla.org/sops/json"
-	sopsyaml "go.mozilla.org/sops/yaml"
+	sopsjson "go.mozilla.org/sops/stores/json"
+	sopsyaml "go.mozilla.org/sops/stores/yaml"
 )
 
 // File is a wrapper around Data that reads a local encrypted
@@ -53,7 +53,7 @@ func Data(data []byte, format string) (cleartext []byte, err error) {
 	if err != nil {
 		return nil, err
 	}
-	tree := sops.Tree{Branch: branch, Metadata: metadata}
+	tree := sops.Tree{Branch: branch, Metadata: *metadata}
 
 	// Decrypt the tree
 	cipher := aes.Cipher{}

example.json

@@ -20,7 +20,7 @@
     ],
     "anEmptyValue": "",
     "sops": {
-        "version": 1.6,
+        "version": "1.6",
         "mac": "ENC[AES256_GCM,data:YlhAZo7NAUHlRQzAPoPha12yl3nEaaq3lKyJ1hoMMtw9M7kts1cfrk301arKHrRAssqcqq1RizYVTyennOIj+TpOYoi8dOzXAtm+NrjQrb7SNosGLPXO2mbvOIoeqzZuzureHRCitNeLwzD8+U/vlGhRTuB6Be7TGt55CD+OMMM=,iv:x9DFpKn9xlq5uoRfbV2rnAsmjY4YyZH6wQnBY80x2dw=,tag:JnGt3eJNLRC3FamfNkAkew==,type:str]",
         "pgp": [
             {

sops.go

@@ -47,8 +47,6 @@ import (
 
 	"go.mozilla.org/sops/keys"
 	"go.mozilla.org/sops/keyservice"
-	"go.mozilla.org/sops/kms"
-	"go.mozilla.org/sops/pgp"
 	"go.mozilla.org/sops/shamir"
 	"golang.org/x/net/context"
 )
@@ -244,6 +242,7 @@ func (tree Tree) Encrypt(key []byte, cipher DataKeyCipher, stash map[string][]in
 
 // Decrypt walks over the tree and decrypts all values with the provided cipher, except those whose key ends with the UnencryptedSuffix specified on the Metadata struct. If decryption is successful, it returns the MAC for the decrypted tree.
 func (tree Tree) Decrypt(key []byte, cipher DataKeyCipher, stash map[string][]interface{}) (string, error) {
+	log.Print("Decrypting SOPS tree")
 	hash := sha512.New()
 	_, err := tree.Branch.walkBranch(tree.Branch, make([]string, 0), func(in interface{}, path []string) (interface{}, error) {
 		var v interface{}
@@ -318,7 +317,7 @@ type KeyGroup []keys.MasterKey
 // Store provides a way to load and save the sops tree along with metadata
 type Store interface {
 	Unmarshal(in []byte) (TreeBranch, error)
-	UnmarshalMetadata(in []byte) (Metadata, error)
+	UnmarshalMetadata(in []byte) (*Metadata, error)
 	Marshal(TreeBranch) ([]byte, error)
 	MarshalWithMetadata(TreeBranch, Metadata) ([]byte, error)
 	MarshalValue(interface{}) ([]byte, error)
@@ -420,45 +419,6 @@ func (m *Metadata) UpdateMasterKeys(dataKey []byte) (errs []error) {
 	})
 }
 
-// ToMap converts the Metadata to a map for serialization purposes
-func (m *Metadata) ToMap() map[string]interface{} {
-	// TODO: This doesn't belong here. This is serialization logic.
-	// It should probably be rewritten so that sops.Metadata gets mapped to some sort of stores.Metadata struct,
-	// which then gets serialized directly
-	out := make(map[string]interface{})
-	out["lastmodified"] = m.LastModified.Format(time.RFC3339)
-	out["unencrypted_suffix"] = m.UnencryptedSuffix
-	out["mac"] = m.MessageAuthenticationCode
-	out["version"] = m.Version
-	out["shamir_quorum"] = m.ShamirQuorum
-	if len(m.KeyGroups) == 1 {
-		for k, v := range m.keyGroupToMap(m.KeyGroups[0]) {
-			out[k] = v
-		}
-	} else {
-		// This is very bad and I should feel bad
-		var groups []map[string][]map[string]interface{}
-		for _, group := range m.KeyGroups {
-			groups = append(groups, m.keyGroupToMap(group))
-		}
-		out["key_groups"] = groups
-	}
-	return out
-}
-
-func (m *Metadata) keyGroupToMap(group KeyGroup) (keys map[string][]map[string]interface{}) {
-	keys = make(map[string][]map[string]interface{})
-	for _, k := range group {
-		switch k := k.(type) {
-		case *pgp.MasterKey:
-			keys["pgp"] = append(keys["pgp"], k.ToMap())
-		case *kms.MasterKey:
-			keys["kms"] = append(keys["kms"], k.ToMap())
-		}
-	}
-	return
-}
-
 // GetDataKeyWithKeyServices retrieves the data key, asking KeyServices to decrypt it with each
 // MasterKey in the Metadata's KeySources until one of them succeeds.
 func (m Metadata) GetDataKeyWithKeyServices(svcs []keyservice.KeyServiceClient) ([]byte, error) {
@@ -531,150 +491,3 @@ func ToBytes(in interface{}) ([]byte, error) {
 		return nil, fmt.Errorf("Could not convert unknown type %T to bytes", in)
 	}
 }
-
-// MapToMetadata tries to convert a map[string]interface{} obtained from an encrypted file into a Metadata struct.
-func MapToMetadata(data map[string]interface{}) (Metadata, error) {
-	// TODO: This doesn't belong here. This is serialization logic.
-	// It should probably be rewritten so that the YAML/JSON gets mapped to a struct which then gets mapped to a
-	// sops.Metadata
-	// TODO: Use KeyGroups
-	var metadata Metadata
-	mac, ok := data["mac"].(string)
-	if !ok {
-		fmt.Println("WARNING: no MAC was found on the input file. " +
-			"Verification will fail. You can use --ignore-mac to skip verification.")
-	}
-	metadata.MessageAuthenticationCode = mac
-	lastModified, err := time.Parse(time.RFC3339, data["lastmodified"].(string))
-	if err != nil {
-		return metadata, fmt.Errorf("Could not parse last modified date: %s", err)
-	}
-	metadata.LastModified = lastModified
-	unencryptedSuffix, ok := data["unencrypted_suffix"].(string)
-	if !ok {
-		unencryptedSuffix = DefaultUnencryptedSuffix
-	}
-	metadata.UnencryptedSuffix = unencryptedSuffix
-	if metadata.Version, ok = data["version"].(string); !ok {
-		metadata.Version = strconv.FormatFloat(data["version"].(float64), 'f', -1, 64)
-	}
-	if shamirQuorum, ok := data["shamir_quorum"].(float64); ok {
-		metadata.ShamirQuorum = int(shamirQuorum)
-	} else if shamirQuorum, ok := data["shamir_quorum"].(int); ok {
-		metadata.ShamirQuorum = shamirQuorum
-	}
-	if keyGroups, ok := data["key_groups"]; ok {
-		var kgs []KeyGroup
-		if gs, ok := keyGroups.([]interface{}); ok {
-			for _, g := range gs {
-				g := g.(map[interface{}]interface{})
-				var group KeyGroup
-				if k, ok := g["kms"].([]interface{}); ok {
-					ks, err := mapKMSEntriesToKeySlice(k)
-					if err == nil {
-						group = append(group, ks...)
-					}
-				}
-				if pgp, ok := g["pgp"].([]interface{}); ok {
-					ks, err := mapPGPEntriesToKeySlice(pgp)
-					if err == nil {
-						group = append(group, ks...)
-					}
-				}
-				kgs = append(kgs, group)
-			}
-		}
-		metadata.KeyGroups = kgs
-	} else {
-		// Old data format, just one KeyGroup
-		var group KeyGroup
-		if k, ok := data["kms"].([]interface{}); ok {
-			ks, err := mapKMSEntriesToKeySlice(k)
-			if err == nil {
-				group = append(group, ks...)
-			}
-		}
-		if pgp, ok := data["pgp"].([]interface{}); ok {
-			ks, err := mapPGPEntriesToKeySlice(pgp)
-			if err == nil {
-				group = append(group, ks...)
-			}
-		}
-		metadata.KeyGroups = []KeyGroup{group}
-	}
-	return metadata, nil
-}
-
-func convertToMapStringInterface(in map[interface{}]interface{}) (map[string]interface{}, error) {
-	// TODO: This doesn't belong here. This is serialization logic.
-	m := make(map[string]interface{})
-	for k, v := range in {
-		key, ok := k.(string)
-		if !ok {
-			return nil, fmt.Errorf("Map contains non-string-key (Type %T): %s", k, k)
-		}
-		m[key] = v
-	}
-	return m, nil
-}
-
-func mapKMSEntriesToKeySlice(in []interface{}) ([]keys.MasterKey, error) {
-	// TODO: This doesn't belong here. This is serialization logic.
-	var keys []keys.MasterKey
-	for _, v := range in {
-		entry, ok := v.(map[string]interface{})
-		if !ok {
-			m, ok := v.(map[interface{}]interface{})
-			var err error
-			entry, err = convertToMapStringInterface(m)
-			if !ok || err != nil {
-				fmt.Println("KMS entry has invalid format, skipping...")
-				continue
-			}
-		}
-		key := &kms.MasterKey{}
-		key.Arn = entry["arn"].(string)
-		key.EncryptedKey = entry["enc"].(string)
-		role, ok := entry["role"].(string)
-		if ok {
-			key.Role = role
-		}
-		creationDate, err := time.Parse(time.RFC3339, entry["created_at"].(string))
-		if err != nil {
-			return keys, fmt.Errorf("Could not parse creation date: %s", err)
-		}
-		if _, ok := entry["context"]; ok {
-			key.EncryptionContext = kms.ParseKMSContext(entry["context"])
-		}
-		key.CreationDate = creationDate
-		keys = append(keys, key)
-	}
-	return keys, nil
-}
-
-func mapPGPEntriesToKeySlice(in []interface{}) ([]keys.MasterKey, error) {
-	// TODO: This doesn't belong here. This is serialization logic.
-	var keys []keys.MasterKey
-	for _, v := range in {
-		entry, ok := v.(map[string]interface{})
-		if !ok {
-			m, ok := v.(map[interface{}]interface{})
-			var err error
-			entry, err = convertToMapStringInterface(m)
-			if !ok || err != nil {
-				fmt.Println("PGP entry has invalid format, skipping...")
-				continue
-			}
-		}
-		key := &pgp.MasterKey{}
-		key.Fingerprint = entry["fp"].(string)
-		key.EncryptedKey = entry["enc"].(string)
-		creationDate, err := time.Parse(time.RFC3339, entry["created_at"].(string))
-		if err != nil {
-			return keys, fmt.Errorf("Could not parse creation date: %s", err)
-		}
-		key.CreationDate = creationDate
-		keys = append(keys, key)
-	}
-	return keys, nil
-}

stores/json/store.go

@@ -1,11 +1,13 @@
-package json //import "go.mozilla.org/sops/json"
+package json //import "go.mozilla.org/sops/stores/json"
 
 import (
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
+
 	"go.mozilla.org/sops"
+	"go.mozilla.org/sops/stores"
 )
 
 // Store handles storage of JSON data.
@@ -52,7 +54,7 @@ func (store BinaryStore) Unmarshal(in []byte) (sops.TreeBranch, error) {
 }
 
 // UnmarshalMetadata takes a binary format sops file and extracts sops' metadata from it
-func (store BinaryStore) UnmarshalMetadata(in []byte) (sops.Metadata, error) {
+func (store BinaryStore) UnmarshalMetadata(in []byte) (*sops.Metadata, error) {
 	return store.store.UnmarshalMetadata(in)
 }
 
@@ -228,7 +230,7 @@ func (store Store) Marshal(tree sops.TreeBranch) ([]byte, error) {
 
 // MarshalWithMetadata takes a sops tree branch and sops metadata and marshals them to json.
 func (store Store) MarshalWithMetadata(tree sops.TreeBranch, metadata sops.Metadata) ([]byte, error) {
-	tree = append(tree, sops.TreeItem{Key: "sops", Value: metadata.ToMap()})
+	tree = append(tree, sops.TreeItem{Key: "sops", Value: stores.MetadataFromInternal(metadata)})
 	out, err := store.jsonFromTreeBranch(tree)
 	if err != nil {
 		return nil, fmt.Errorf("Error marshaling to json: %s", err)
@@ -246,14 +248,11 @@ func (store Store) MarshalValue(v interface{}) ([]byte, error) {
 }
 
 // UnmarshalMetadata takes a json string and extracts sops' metadata from it
-func (store Store) UnmarshalMetadata(in []byte) (sops.Metadata, error) {
-	var ok bool
-	data := make(map[string]interface{})
-	if err := json.Unmarshal(in, &data); err != nil {
-		return sops.Metadata{}, fmt.Errorf("Error unmarshalling input json: %s", err)
+func (store Store) UnmarshalMetadata(in []byte) (*sops.Metadata, error) {
+	file := stores.SopsFile{}
+	err := json.Unmarshal(in, &file)
+	if err != nil {
+		return nil, fmt.Errorf("Error unmarshalling input json: %s", err)
 	}
-	if data, ok = data["sops"].(map[string]interface{}); !ok {
-		return sops.Metadata{}, sops.MetadataNotFound
-	}
-	return sops.MapToMetadata(data)
-}
+	return file.Metadata.ToInternal()
+}

stores/stores.go

@@ -0,0 +1,192 @@
+package stores
+
+import (
+	"time"
+
+	"fmt"
+
+	"go.mozilla.org/sops"
+	"go.mozilla.org/sops/kms"
+	"go.mozilla.org/sops/pgp"
+)
+
+type SopsFile struct {
+	Data     interface{} `yaml:"data" json:"data"`
+	Metadata Metadata    `yaml:"sops" json:"sops"`
+}
+
+// metadata is stored in SOPS encrypted files, and it contains the information necessary to decrypt the file.
+// This struct is just used for serialization, and SOPS uses another struct internally, sops.Metadata. It exists
+// in order to allow the binary format to stay backwards compatible over time, but at the same time allow the internal
+// representation SOPS uses to change over time.
+type Metadata struct {
+	LastModified              string     `yaml:"lastmodified" json:"lastmodified"`
+	UnencryptedSuffix         string     `yaml:"unencrypted_suffix" json:"unencrypted_suffix"`
+	MessageAuthenticationCode string     `yaml:"mac" json:"mac"`
+	Version                   string     `yaml:"version" json:"version"`
+	ShamirQuorum              int        `yaml:"shamir_quorum,omitempty" json:"shamir_quorum,omitempty"`
+	KeyGroups                 []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty"`
+	PGPKeys                   []pgpkey   `yaml:"pgp,omitempty" json:"pgp,omitempty"`
+	KMSKeys                   []kmskey   `yaml:"kms,omitempty" json:"kms,omitempty"`
+}
+
+type keygroup struct {
+	PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"`
+	KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"`
+}
+
+type pgpkey struct {
+	CreatedAt        string `yaml:"created_at" json:"created_at"`
+	EncryptedDataKey string `yaml:"enc" json:"enc"`
+	Fingerprint      string `yaml:"fp" json:"fp"`
+}
+
+type kmskey struct {
+	CreatedAt        string             `yaml:"created_at" json:"created_at"`
+	EncryptedDataKey string             `yaml:"enc" json:"enc"`
+	Arn              string             `yaml:"arn" json:"arn"`
+	Role             string             `yaml:"role" json:"role"`
+	Context          map[string]*string `yaml:"context" json:"context"`
+}
+
+func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata {
+	var m Metadata
+	m.LastModified = sopsMetadata.LastModified.Format(time.RFC3339)
+	m.UnencryptedSuffix = sopsMetadata.UnencryptedSuffix
+	m.MessageAuthenticationCode = sopsMetadata.MessageAuthenticationCode
+	m.Version = sopsMetadata.Version
+	m.ShamirQuorum = sopsMetadata.ShamirQuorum
+	if len(sopsMetadata.KeyGroups) == 1 {
+		group := sopsMetadata.KeyGroups[0]
+		m.PGPKeys = pgpKeysFromGroup(group)
+		m.KMSKeys = kmsKeysFromGroup(group)
+	} else {
+		for _, group := range sopsMetadata.KeyGroups {
+			m.KeyGroups = append(m.KeyGroups, keygroup{
+				KMSKeys: kmsKeysFromGroup(group),
+				PGPKeys: pgpKeysFromGroup(group),
+			})
+		}
+	}
+	return m
+}
+
+func pgpKeysFromGroup(group sops.KeyGroup) (keys []pgpkey) {
+	for _, key := range group {
+		switch key := key.(type) {
+		case *pgp.MasterKey:
+			keys = append(keys, pgpkey{
+				Fingerprint:      key.Fingerprint,
+				EncryptedDataKey: key.EncryptedKey,
+				CreatedAt:        key.CreationDate.Format(time.RFC3339),
+			})
+		}
+	}
+	return
+}
+
+func kmsKeysFromGroup(group sops.KeyGroup) (keys []kmskey) {
+	for _, key := range group {
+		switch key := key.(type) {
+		case *kms.MasterKey:
+			keys = append(keys, kmskey{
+				Arn:              key.Arn,
+				CreatedAt:        key.CreationDate.Format(time.RFC3339),
+				EncryptedDataKey: key.EncryptedKey,
+				Context:          key.EncryptionContext,
+				Role:             key.Role,
+			})
+		}
+	}
+	return
+}
+
+func (m *Metadata) ToInternal() (*sops.Metadata, error) {
+	lastModified, err := time.Parse(time.RFC3339, m.LastModified)
+	if err != nil {
+		return nil, err
+	}
+	groups, err := m.internalKeygroups()
+	if err != nil {
+		return nil, err
+	}
+	if m.UnencryptedSuffix == "" {
+		m.UnencryptedSuffix = sops.DefaultUnencryptedSuffix
+	}
+	return &sops.Metadata{
+		KeyGroups:                 groups,
+		ShamirQuorum:              m.ShamirQuorum,
+		Version:                   m.Version,
+		MessageAuthenticationCode: m.MessageAuthenticationCode,
+		UnencryptedSuffix:         m.UnencryptedSuffix,
+		LastModified:              lastModified,
+	}, nil
+}
+
+func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey) (sops.KeyGroup, error) {
+	var internalGroup sops.KeyGroup
+	for _, kmsKey := range kmsKeys {
+		k, err := kmsKey.toInternal()
+		if err != nil {
+			return nil, err
+		}
+		internalGroup = append(internalGroup, k)
+	}
+	for _, pgpKey := range pgpKeys {
+		k, err := pgpKey.toInternal()
+		if err != nil {
+			return nil, err
+		}
+		internalGroup = append(internalGroup, k)
+	}
+	return internalGroup, nil
+}
+
+func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) {
+	var internalGroups []sops.KeyGroup
+	if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 {
+		internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys)
+		if err != nil {
+			return nil, err
+		}
+		internalGroups = append(internalGroups, internalGroup)
+		return internalGroups, nil
+	} else if len(m.KeyGroups) > 0 {
+		for _, group := range m.KeyGroups {
+			internalGroup, err := internalGroupFrom(group.KMSKeys, group.PGPKeys)
+			if err != nil {
+				return nil, err
+			}
+			internalGroups = append(internalGroups, internalGroup)
+		}
+		return internalGroups, nil
+	} else {
+		return nil, fmt.Errorf("No keys found in file")
+	}
+}
+
+func (kmsKey *kmskey) toInternal() (*kms.MasterKey, error) {
+	creationDate, err := time.Parse(time.RFC3339, kmsKey.CreatedAt)
+	if err != nil {
+		return nil, err
+	}
+	return &kms.MasterKey{
+		Role:              kmsKey.Role,
+		EncryptionContext: kmsKey.Context,
+		EncryptedKey:      kmsKey.EncryptedDataKey,
+		CreationDate:      creationDate,
+		Arn:               kmsKey.Arn,
+	}, nil
+}
+
+func (pgpKey *pgpkey) toInternal() (*pgp.MasterKey, error) {
+	creationDate, err := time.Parse(time.RFC3339, pgpKey.CreatedAt)
+	if err != nil {
+		return nil, err
+	}
+	return &pgp.MasterKey{
+		EncryptedKey: pgpKey.EncryptedDataKey,
+		CreationDate: creationDate,
+		Fingerprint:  pgpKey.Fingerprint,
+	}, nil
+}

stores/yaml/store.go

@@ -1,9 +1,11 @@
-package yaml //import "go.mozilla.org/sops/yaml"
+package yaml //import "go.mozilla.org/sops/stores/yaml"
 
 import (
 	"fmt"
+
 	"github.com/mozilla-services/yaml"
 	"go.mozilla.org/sops"
+	"go.mozilla.org/sops/stores"
 )
 
 // Store handles storage of YAML data
@@ -124,7 +126,7 @@ func (store Store) Marshal(tree sops.TreeBranch) ([]byte, error) {
 // MarshalWithMetadata takes a sops tree branch and metadata and marshals them into a yaml document
 func (store Store) MarshalWithMetadata(tree sops.TreeBranch, metadata sops.Metadata) ([]byte, error) {
 	yamlMap := store.treeBranchToYamlMap(tree)
-	yamlMap = append(yamlMap, yaml.MapItem{Key: "sops", Value: metadata.ToMap()})
+	yamlMap = append(yamlMap, yaml.MapItem{Key: "sops", Value: stores.MetadataFromInternal(metadata)})
 	out, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap)
 	if err != nil {
 		return nil, fmt.Errorf("Error marshaling to yaml: %s", err)
@@ -138,23 +140,12 @@ func (store Store) MarshalValue(v interface{}) ([]byte, error) {
 	return (&yaml.YAMLMarshaler{Indent: 4}).Marshal(v)
 }
 
-
 // UnmarshalMetadata takes a yaml document as a string and extracts sops' metadata from it
-func (store *Store) UnmarshalMetadata(in []byte) (sops.Metadata, error) {
-	var ok bool
-	data := make(map[interface{}]interface{})
-	err := yaml.Unmarshal(in, &data)
+func (s *Store) UnmarshalMetadata(in []byte) (*sops.Metadata, error) {
+	file := stores.SopsFile{}
+	err := yaml.Unmarshal(in, &file)
 	if err != nil {
-		return sops.Metadata{}, fmt.Errorf("Error unmarshalling input yaml: %s", err)
+		return nil, fmt.Errorf("Error unmarshalling input yaml: %s", err)
 	}
-	data, ok = data["sops"].(map[interface{}]interface{})
-	if !ok {
-		return sops.Metadata{}, sops.MetadataNotFound
-	}
-	metadataBranch := make(map[string]interface{})
-	for k, v := range data {
-		key, _ := k.(string)
-		metadataBranch[key] = v
-	}
-	return sops.MapToMetadata(metadataBranch)
-}
+	return file.Metadata.ToInternal()
+}