Add support for --unencrypted-regex (#715)

* Add support for --unencrypted-regex

* Fix grammar mistake

* Add gofmt'd files

README.rst

@@ -1426,9 +1426,20 @@ will encrypt the values under the ``data`` and ``stringData`` keys in a YAML fil
 containing kubernetes secrets.  It will not encrypt other values that help you to
 navigate the file, like ``metadata`` which contains the secrets' names.
 
+Conversely, you can opt in to only left certain keys without encrypting by using the 
+``--unencrypted-regex`` option, which will leave the values unencrypted of those keys 
+that match the supplied regular expression. For example, this command:
+
+.. code:: bash
+
+  $ sops --encrypt --unencrypted-regex '^(description|metadata)$' k8s-secrets.yaml
+
+will not encrypt the values under the ``description`` and ``metadata`` keys in a YAML file
+containing kubernetes secrets, while encrypting everything else.
+
 You can also specify these options in the ``.sops.yaml`` config file.
 
-Note: these three options ``--unencrypted-suffix``, ``--encrypted-suffix``, and ``--encrypted-regex`` are
+Note: these four options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are
 mutually exclusive and cannot all be used in the same file.
 
 Encryption Protocol

cmd/sops/edit.go

@@ -37,6 +37,7 @@ type editExampleOpts struct {
 	editOpts
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	KeyGroups         []sops.KeyGroup
 	GroupThreshold    int
@@ -66,6 +67,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
+			UnencryptedRegex:  opts.UnencryptedRegex,
 			EncryptedRegex:    opts.EncryptedRegex,
 			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,
@@ -132,7 +134,7 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 	if err != nil {
 		return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
 	}
-	
+
 	// Close temporary file, since Windows won't delete the file unless it's closed beforehand
 	defer tmpfile.Close()
 

cmd/sops/encrypt.go

@@ -22,6 +22,7 @@ type encryptOpts struct {
 	KeyServices       []keyservice.KeyServiceClient
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	KeyGroups         []sops.KeyGroup
 	GroupThreshold    int
@@ -77,6 +78,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
+			UnencryptedRegex:  opts.UnencryptedRegex,
 			EncryptedRegex:    opts.EncryptedRegex,
 			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,

cmd/sops/main.go

@@ -185,9 +185,9 @@ func main() {
 					Usage: "the user to run the command as",
 				},
 				cli.StringFlag{
-                                        Name:  "input-type",
-                                        Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type",
-                                },
+					Name:  "input-type",
+					Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type",
+				},
 				cli.StringFlag{
 					Name:  "output-type",
 					Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format",
@@ -625,6 +625,10 @@ func main() {
 			Name:  "encrypted-suffix",
 			Usage: "override the encrypted key suffix. When empty, all keys will be encrypted, unless otherwise marked with unencrypted-suffix.",
 		},
+		cli.StringFlag{
+			Name:  "unencrypted-regex",
+			Usage: "set the unencrypted key suffix. When specified, only keys matching the regex will be left unencrypted.",
+		},
 		cli.StringFlag{
 			Name:  "encrypted-regex",
 			Usage: "set the encrypted key suffix. When specified, only keys matching the regex will be encrypted.",
@@ -682,6 +686,7 @@ func main() {
 		unencryptedSuffix := c.String("unencrypted-suffix")
 		encryptedSuffix := c.String("encrypted-suffix")
 		encryptedRegex := c.String("encrypted-regex")
+		unencryptedRegex := c.String("unencrypted-regex")
 		conf, err := loadConfig(c, fileName, nil)
 		if err != nil {
 			return toExitError(err)
@@ -697,6 +702,9 @@ func main() {
 			if encryptedRegex == "" {
 				encryptedRegex = conf.EncryptedRegex
 			}
+			if unencryptedRegex == "" {
+				unencryptedRegex = conf.UnencryptedRegex
+			}
 		}
 
 		cryptRuleCount := 0
@@ -709,9 +717,12 @@ func main() {
 		if encryptedRegex != "" {
 			cryptRuleCount++
 		}
+		if unencryptedRegex != "" {
+			cryptRuleCount++
+		}
 
 		if cryptRuleCount > 1 {
-			return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file", codes.ErrorConflictingParameters)
+			return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex or unencrypted_regex in the same file", codes.ErrorConflictingParameters)
 		}
 
 		// only supply the default UnencryptedSuffix when EncryptedSuffix and EncryptedRegex are not provided
@@ -742,6 +753,7 @@ func main() {
 				Cipher:            aes.NewCipher(),
 				UnencryptedSuffix: unencryptedSuffix,
 				EncryptedSuffix:   encryptedSuffix,
+				UnencryptedRegex:  unencryptedRegex,
 				EncryptedRegex:    encryptedRegex,
 				KeyServices:       svcs,
 				KeyGroups:         groups,
@@ -879,6 +891,7 @@ func main() {
 					editOpts:          opts,
 					UnencryptedSuffix: unencryptedSuffix,
 					EncryptedSuffix:   encryptedSuffix,
+					UnencryptedRegex:  unencryptedRegex,
 					EncryptedRegex:    encryptedRegex,
 					KeyGroups:         groups,
 					GroupThreshold:    threshold,

config/config.go

@@ -117,6 +117,7 @@ type creationRule struct {
 	ShamirThreshold   int        `yaml:"shamir_threshold"`
 	UnencryptedSuffix string     `yaml:"unencrypted_suffix"`
 	EncryptedSuffix   string     `yaml:"encrypted_suffix"`
+	UnencryptedRegex  string     `yaml:"unencrypted_regex"`
 	EncryptedRegex    string     `yaml:"encrypted_regex"`
 }
 
@@ -135,6 +136,7 @@ type Config struct {
 	ShamirThreshold   int
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	Destination       publish.Destination
 	OmitExtensions    bool
@@ -235,6 +237,7 @@ func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string)
 		ShamirThreshold:   rule.ShamirThreshold,
 		UnencryptedSuffix: rule.UnencryptedSuffix,
 		EncryptedSuffix:   rule.EncryptedSuffix,
+		UnencryptedRegex:  rule.UnencryptedRegex,
 		EncryptedRegex:    rule.EncryptedRegex,
 	}, nil
 }

config/config_test.go

@@ -137,6 +137,7 @@ creation_rules:
     kms: "1"
     pgp: "2"
     encrypted_regex: "^enc:"
+    unencrypted_regex: "^dec:"
     `)
 
 var sampleConfigWithInvalidParameters = []byte(`
@@ -349,6 +350,12 @@ func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) {
 	assert.Equal(t, "_enc", conf.EncryptedSuffix)
 }
 
+func TestLoadConfigFileWithUnencryptedRegex(t *testing.T) {
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "^dec:", conf.UnencryptedRegex)
+}
+
 func TestLoadConfigFileWithEncryptedRegex(t *testing.T) {
 	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
 	assert.Equal(t, nil, err)

hcvault/keysource.go

@@ -29,7 +29,7 @@ func init() {
 type MasterKey struct {
 	EncryptedKey string
 	KeyName      string
-	EnginePath  string
+	EnginePath   string
 	VaultAddress string
 	CreationDate time.Time
 }
@@ -100,7 +100,7 @@ func getBackendAndKeyFromPath(fullPath string) (enginePath, keyName string, err
 func NewMasterKey(addess, enginePath, keyName string) *MasterKey {
 	mk := &MasterKey{
 		VaultAddress: addess,
-		EnginePath:  enginePath,
+		EnginePath:   enginePath,
 		KeyName:      keyName,
 		CreationDate: time.Now().UTC(),
 	}

keyservice/keyservice.go

@@ -39,8 +39,8 @@ func KeyFromMasterKey(mk keys.MasterKey) Key {
 			KeyType: &Key_VaultKey{
 				VaultKey: &VaultKey{
 					VaultAddress: mk.VaultAddress,
-					EnginePath:    mk.EnginePath,
-					KeyName:        mk.KeyName,
+					EnginePath:   mk.EnginePath,
+					KeyName:      mk.KeyName,
 				},
 			},
 		}

keyservice/keyservice.pb.go

@@ -293,7 +293,7 @@ func (m *GcpKmsKey) GetResourceId() string {
 
 type VaultKey struct {
 	VaultAddress         string   `protobuf:"bytes,1,opt,name=vault_address,json=vaultAddress,proto3" json:"vault_address,omitempty"`
-	EnginePath          string   `protobuf:"bytes,2,opt,name=engine_path,json=enginePath,proto3" json:"engine_path,omitempty"`
+	EnginePath           string   `protobuf:"bytes,2,opt,name=engine_path,json=enginePath,proto3" json:"engine_path,omitempty"`
 	KeyName              string   `protobuf:"bytes,3,opt,name=key_name,json=keyName,proto3" json:"key_name,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`

sops.go

@@ -287,9 +287,10 @@ func (branch TreeBranch) walkBranch(in TreeBranch, path []string, onLeaves func(
 // Encrypt walks over the tree and encrypts all values with the provided cipher,
 // except those whose key ends with the UnencryptedSuffix specified on the
 // Metadata struct, those not ending with EncryptedSuffix, if EncryptedSuffix
-// is provided (by default it is not), or those not matching EncryptedRegex,
-// if EncryptedRegex is provided (by default it is not).  If encryption is
-// successful, it returns the MAC for the encrypted tree.
+// is provided (by default it is not), those not matching EncryptedRegex,
+// if EncryptedRegex is provided (by default it is not) or those matching
+// UnencryptedRegex, if UnencryptedRegex is provided (by default it is not).
+// If encryption is successful, it returns the MAC for the encrypted tree.
 func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) {
 	audit.SubmitEvent(audit.EncryptEvent{
 		File: tree.FilePath,
@@ -323,6 +324,15 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) {
 					}
 				}
 			}
+			if tree.Metadata.UnencryptedRegex != "" {
+				for _, p := range path {
+					matched, _ := regexp.Match(tree.Metadata.UnencryptedRegex, []byte(p))
+					if matched {
+						encrypted = false
+						break
+					}
+				}
+			}
 			if tree.Metadata.EncryptedRegex != "" {
 				encrypted = false
 				for _, p := range path {
@@ -358,7 +368,8 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) {
 // Decrypt walks over the tree and decrypts all values with the provided cipher,
 // except those whose key ends with the UnencryptedSuffix specified on the Metadata struct,
 // those not ending with EncryptedSuffix, if EncryptedSuffix is provided (by default it is not),
-// or those not matching EncryptedRegex, if EncryptedRegex is provided (by default it is not).
+// those not matching EncryptedRegex, if EncryptedRegex is provided (by default it is not),
+// or those matching UnencryptedRegex, if UnencryptedRegex is provided (by default it is not).
 // If decryption is successful, it returns the MAC for the decrypted tree.
 func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) {
 	log.Debug("Decrypting tree")
@@ -386,6 +397,15 @@ func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) {
 					}
 				}
 			}
+			if tree.Metadata.UnencryptedRegex != "" {
+				for _, p := range path {
+					matched, _ := regexp.Match(tree.Metadata.UnencryptedRegex, []byte(p))
+					if matched {
+						encrypted = false
+						break
+					}
+				}
+			}
 			if tree.Metadata.EncryptedRegex != "" {
 				encrypted = false
 				for _, p := range path {
@@ -466,6 +486,7 @@ type Metadata struct {
 	LastModified              time.Time
 	UnencryptedSuffix         string
 	EncryptedSuffix           string
+	UnencryptedRegex          string
 	EncryptedRegex            string
 	MessageAuthenticationCode string
 	Version                   string

sops_test.go

@@ -189,6 +189,59 @@ func TestEncryptedRegex(t *testing.T) {
 	}
 }
 
+func TestUnencryptedRegex(t *testing.T) {
+	branches := TreeBranches{
+		TreeBranch{
+			TreeItem{
+				Key:   "dec:foo",
+				Value: "bar",
+			},
+			TreeItem{
+				Key: "dec:bar",
+				Value: TreeBranch{
+					TreeItem{
+						Key:   "foo",
+						Value: "bar",
+					},
+				},
+			},
+		},
+	}
+	tree := Tree{Branches: branches, Metadata: Metadata{UnencryptedRegex: "^dec:"}}
+	expected := TreeBranch{
+		TreeItem{
+			Key:   "dec:foo",
+			Value: "bar",
+		},
+		TreeItem{
+			Key: "dec:bar",
+			Value: TreeBranch{
+				TreeItem{
+					Key:   "foo",
+					Value: "bar",
+				},
+			},
+		},
+	}
+	cipher := reverseCipher{}
+
+	_, err := tree.Encrypt(bytes.Repeat([]byte("f"), 32), cipher)
+	if err != nil {
+		t.Errorf("Encrypting the tree failed: %s", err)
+	}
+	// expected[1].Value[] = "bar"
+	if !reflect.DeepEqual(tree.Branches[0], expected) {
+		t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branches[0], expected)
+	}
+	_, err = tree.Decrypt(bytes.Repeat([]byte("f"), 32), cipher)
+	if err != nil {
+		t.Errorf("Decrypting the tree failed: %s", err)
+	}
+	if !reflect.DeepEqual(tree.Branches[0], expected) {
+		t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branches[0], expected)
+	}
+}
+
 type MockCipher struct{}
 
 func (m MockCipher) Encrypt(value interface{}, key []byte, path string) (string, error) {

stores/dotenv/store_test.go

@@ -62,4 +62,4 @@ func TestEmitValueString(t *testing.T) {
 func TestEmitValueNonstring(t *testing.T) {
 	_, err := (&Store{}).EmitValue(BRANCH)
 	assert.NotNil(t, err)
-}
+}

stores/stores.go

@@ -47,6 +47,7 @@ type Metadata struct {
 	PGPKeys                   []pgpkey    `yaml:"pgp" json:"pgp"`
 	UnencryptedSuffix         string      `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"`
 	EncryptedSuffix           string      `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"`
+	UnencryptedRegex          string      `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty"`
 	EncryptedRegex            string      `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"`
 	Version                   string      `yaml:"version" json:"version"`
 }
@@ -102,6 +103,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata {
 	m.LastModified = sopsMetadata.LastModified.Format(time.RFC3339)
 	m.UnencryptedSuffix = sopsMetadata.UnencryptedSuffix
 	m.EncryptedSuffix = sopsMetadata.EncryptedSuffix
+	m.UnencryptedRegex = sopsMetadata.UnencryptedRegex
 	m.EncryptedRegex = sopsMetadata.EncryptedRegex
 	m.MessageAuthenticationCode = sopsMetadata.MessageAuthenticationCode
 	m.Version = sopsMetadata.Version
@@ -222,12 +224,15 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) {
 	if m.EncryptedSuffix != "" {
 		cryptRuleCount++
 	}
+	if m.UnencryptedRegex != "" {
+		cryptRuleCount++
+	}
 	if m.EncryptedRegex != "" {
 		cryptRuleCount++
 	}
 
 	if cryptRuleCount > 1 {
-		return sops.Metadata{}, fmt.Errorf("Cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file")
+		return sops.Metadata{}, fmt.Errorf("Cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex or unencrypted_regex in the same file")
 	}
 
 	if cryptRuleCount == 0 {
@@ -240,6 +245,7 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) {
 		MessageAuthenticationCode: m.MessageAuthenticationCode,
 		UnencryptedSuffix:         m.UnencryptedSuffix,
 		EncryptedSuffix:           m.EncryptedSuffix,
+		UnencryptedRegex:          m.UnencryptedRegex,
 		EncryptedRegex:            m.EncryptedRegex,
 		LastModified:              lastModified,
 	}, nil