diff --git a/README.rst b/README.rst index cea6ce80c..042999af3 100644 --- a/README.rst +++ b/README.rst @@ -1471,6 +1471,25 @@ The value must be formatted as json. $ sops set ~/git/svc/sops/example.yaml '["an_array"][1]' '{"uid1":null,"uid2":1000,"uid3":["bob"]}' +Unset a sub-part in a document tree +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Symmetrically, SOPS can unset a specific part of a YAML or JSON document, by providing +the path in the ``unset`` command. This is useful to unset specific values, like keys, without +needing an editor. + +.. code:: sh + + $ sops unset ~/git/svc/sops/example.yaml '["app2"]["key"]' + +The tree path syntax uses regular python dictionary syntax, without the +variable name. Set to keys by naming them, and array elements by +numbering them. + +.. code:: sh + + $ sops unset ~/git/svc/sops/example.yaml '["an_array"][1]' + Showing diffs in cleartext in git ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 321a800a5..bc888428d 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -1334,6 +1334,94 @@ func main() { return toExitError(err) } + // We open the file *after* the operations on the tree have been + // executed to avoid truncating it when there's errors + file, err := os.Create(fileName) + if err != nil { + return common.NewExitError(fmt.Sprintf("Could not open in-place file for writing: %s", err), codes.CouldNotWriteOutputFile) + } + defer file.Close() + _, err = file.Write(output) + if err != nil { + return toExitError(err) + } + log.Info("File written successfully") + return nil + }, + }, + { + Name: "unset", + Usage: `unset a specific key or branch in the input document.`, + ArgsUsage: `file index`, + Flags: append([]cli.Flag{ + cli.StringFlag{ + Name: "input-type", + Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + }, + cli.StringFlag{ + Name: "output-type", + Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format", + }, + cli.IntFlag{ + Name: "shamir-secret-sharing-threshold", + Usage: "the number of master keys required to retrieve the data key with shamir", + }, + cli.BoolFlag{ + Name: "ignore-mac", + Usage: "ignore Message Authentication Code during decryption", + }, + cli.StringFlag{ + Name: "decryption-order", + Usage: "comma separated list of decryption key types", + EnvVar: "SOPS_DECRYPTION_ORDER", + }, + cli.BoolFlag{ + Name: "idempotent", + Usage: "do nothing if the given index does not exist", + }, + }, keyserviceFlags...), + Action: func(c *cli.Context) error { + if c.Bool("verbose") { + logging.SetLevel(logrus.DebugLevel) + } + if c.NArg() != 2 { + return common.NewExitError("Error: no file specified, or index is missing", codes.NoFileSpecified) + } + fileName, err := filepath.Abs(c.Args()[0]) + if err != nil { + return toExitError(err) + } + + inputStore := inputStore(c, fileName) + outputStore := outputStore(c, fileName) + svcs := keyservices(c) + + path, err := parseTreePath(c.Args()[1]) + if err != nil { + return common.NewExitError("Invalid unset index format", codes.ErrorInvalidSetFormat) + } + + order, err := decryptionOrder(c.String("decryption-order")) + if err != nil { + return toExitError(err) + } + output, err := unset(unsetOpts{ + OutputStore: outputStore, + InputStore: inputStore, + InputPath: fileName, + Cipher: aes.NewCipher(), + KeyServices: svcs, + DecryptionOrder: order, + IgnoreMAC: c.Bool("ignore-mac"), + TreePath: path, + }) + if err != nil { + if _, ok := err.(*sops.SopsKeyNotFound); ok && c.Bool("idempotent") { + return nil + } + return toExitError(err) + } + // We open the file *after* the operations on the tree have been // executed to avoid truncating it when there's errors file, err := os.Create(fileName) diff --git a/cmd/sops/unset.go b/cmd/sops/unset.go new file mode 100644 index 000000000..bb369748c --- /dev/null +++ b/cmd/sops/unset.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + + "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/cmd/sops/codes" + "github.com/getsops/sops/v3/cmd/sops/common" + "github.com/getsops/sops/v3/keyservice" +) + +type unsetOpts struct { + Cipher sops.Cipher + InputStore sops.Store + OutputStore sops.Store + InputPath string + IgnoreMAC bool + TreePath []interface{} + KeyServices []keyservice.KeyServiceClient + DecryptionOrder []string +} + +func unset(opts unsetOpts) ([]byte, error) { + // Load the file + tree, err := common.LoadEncryptedFileWithBugFixes(common.GenericDecryptOpts{ + Cipher: opts.Cipher, + InputStore: opts.InputStore, + InputPath: opts.InputPath, + IgnoreMAC: opts.IgnoreMAC, + KeyServices: opts.KeyServices, + }) + if err != nil { + return nil, err + } + + // Decrypt the file + dataKey, err := common.DecryptTree(common.DecryptTreeOpts{ + Cipher: opts.Cipher, + IgnoreMac: opts.IgnoreMAC, + Tree: tree, + KeyServices: opts.KeyServices, + DecryptionOrder: opts.DecryptionOrder, + }) + if err != nil { + return nil, err + } + + // Unset the value + newBranch, err := tree.Branches[0].Unset(opts.TreePath) + if err != nil { + return nil, err + } + tree.Branches[0] = newBranch + + err = common.EncryptTree(common.EncryptTreeOpts{ + DataKey: dataKey, Tree: tree, Cipher: opts.Cipher, + }) + if err != nil { + return nil, err + } + + encryptedFile, err := opts.OutputStore.EmitEncryptedFile(*tree) + if err != nil { + return nil, common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree) + } + return encryptedFile, err +} diff --git a/functional-tests/src/lib.rs b/functional-tests/src/lib.rs index b9691a1d3..cca673edb 100644 --- a/functional-tests/src/lib.rs +++ b/functional-tests/src/lib.rs @@ -273,12 +273,12 @@ bar: baz", .arg(r#"{"aa": "aaa"}"#) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -317,12 +317,12 @@ bar: baz", .arg(r#"{"cc": "ccc"}"#) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -365,12 +365,12 @@ b: ba"# .arg(r#"{"aa": "aaa"}"#) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -413,12 +413,12 @@ b: ba"# .arg(r#"{"cc": "ccc"}"#) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -472,12 +472,12 @@ b: ba"# .arg(file_path.clone()) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -529,12 +529,12 @@ b: ba"# .arg(file_path.clone()) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); let mut s = String::new(); File::open(file_path) .unwrap() @@ -549,6 +549,224 @@ b: ba"# } } + #[test] + fn unset_json_file() { + // Test removal of tree branch + let file_path = + prepare_temp_file("test_unset.json", r#"{"a": 2, "b": "ba", "c": [1,2]}"#.as_bytes()); + assert!( + Command::new(SOPS_BINARY_PATH) + .arg("encrypt") + .arg("-i") + .arg(file_path.clone()) + .output() + .expect("Error running sops") + .status + .success(), + "sops didn't exit successfully" + ); + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg(file_path.clone()) + .arg(r#"["a"]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut content) + .unwrap(); + let data: Value = serde_json::from_str(&content).expect("Error parsing sops's JSON output"); + if let Value::Mapping(data) = data { + assert!(!data.contains_key(&Value::String("a".to_owned()))); + assert!(data.contains_key(&Value::String("b".to_owned()))); + } else { + panic!("Output JSON does not have the expected structure"); + } + + // Test idempotent unset + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg("--idempotent") + .arg(file_path.clone()) + .arg(r#"["a"]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut idempotent_content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut idempotent_content) + .unwrap(); + assert!(idempotent_content == content); + + // Test removal of list item + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg(file_path.clone()) + .arg(r#"["c"][1]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut content) + .unwrap(); + let data: Value = serde_json::from_str(&content).expect("Error parsing sops's JSON output"); + if let Value::Mapping(data) = data { + assert_eq!(data["c"].as_sequence().unwrap().len(), 1); + } else { + panic!("Output JSON does not have the expected structure"); + } + + // Test idempotent unset list item + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg("--idempotent") + .arg(file_path.clone()) + .arg(r#"["c"][1]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut idempotent_content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut idempotent_content) + .unwrap(); + assert!(idempotent_content == content); + } + + #[test] + fn unset_yaml_file() { + // Test removal of tree branch + let file_path = + prepare_temp_file("test_unset.yaml", r#"{"a": 2, "b": "ba", "c": [1,2]}"#.as_bytes()); + assert!( + Command::new(SOPS_BINARY_PATH) + .arg("encrypt") + .arg("-i") + .arg(file_path.clone()) + .output() + .expect("Error running sops") + .status + .success(), + "sops didn't exit successfully" + ); + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg(file_path.clone()) + .arg(r#"["a"]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut content) + .unwrap(); + let data: Value = serde_yaml::from_str(&content).expect("Error parsing sops's YAML output"); + if let Value::Mapping(data) = data { + assert!(!data.contains_key(&Value::String("a".to_owned()))); + assert!(data.contains_key(&Value::String("b".to_owned()))); + } else { + panic!("Output YAML does not have the expected structure"); + } + + // Test idempotent unset + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg("--idempotent") + .arg(file_path.clone()) + .arg(r#"["a"]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut idempotent_content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut idempotent_content) + .unwrap(); + assert!(idempotent_content == content); + + // Test removal of list item + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg(file_path.clone()) + .arg(r#"["c"][0]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut content) + .unwrap(); + let data: Value = serde_yaml::from_str(&content).expect("Error parsing sops's YAML output"); + if let Value::Mapping(data) = data { + assert_eq!(data["c"].as_sequence().unwrap().len(), 1); + } else { + panic!("Output YAML does not have the expected structure"); + } + + // Test idempotent unset list item + let output = Command::new(SOPS_BINARY_PATH) + .arg("unset") + .arg("--idempotent") + .arg(file_path.clone()) + .arg(r#"["c"][1]"#) + .output() + .expect("Error running sops"); + println!( + "stdout: {}, stderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.status.success(), "sops didn't exit successfully"); + let mut idempotent_content = String::new(); + File::open(file_path.clone()) + .unwrap() + .read_to_string(&mut idempotent_content) + .unwrap(); + assert!(idempotent_content == content); + } + #[test] fn decrypt_file_no_mac() { let file_path = prepare_temp_file( @@ -876,12 +1094,12 @@ echo -E "${foo}" .arg(format!("/bin/bash {}", print_foo)) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); assert_eq!(String::from_utf8_lossy(&output.stdout), "bar\n"); let print_bar = prepare_temp_file( "print_bar.sh", @@ -896,12 +1114,12 @@ echo -E "${bar}" .arg(format!("/bin/bash {}", print_bar)) .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); assert_eq!(String::from_utf8_lossy(&output.stdout), "baz\nbam\n"); } @@ -935,12 +1153,12 @@ bar: |- .arg("cat {}") .output() .expect("Error running sops"); - assert!(output.status.success(), "sops didn't exit successfully"); println!( "stdout: {}, stderr: {}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + assert!(output.status.success(), "sops didn't exit successfully"); assert_eq!( String::from_utf8_lossy(&output.stdout), r#"{ diff --git a/sops.go b/sops.go index c1433425c..44e5b14f1 100644 --- a/sops.go +++ b/sops.go @@ -42,6 +42,7 @@ import ( "fmt" "reflect" "regexp" + "slices" "sort" "strconv" "strings" @@ -76,6 +77,15 @@ const MacMismatch = sopsError("MAC mismatch") // MetadataNotFound occurs when the input file is malformed and doesn't have sops metadata in it const MetadataNotFound = sopsError("sops metadata not found") +type SopsKeyNotFound struct { + Key interface{} + Msg string +} + +func (e *SopsKeyNotFound) Error() string { + return fmt.Sprintf(e.Msg, e.Key) +} + // MACOnlyEncryptedInitialization is a constant and known sequence of 32 bytes used to initialize // MAC which is computed only over values which end up encrypted. That assures that a MAC with the // setting enabled is always different from a MAC with this setting disabled. @@ -189,6 +199,53 @@ func (branch TreeBranch) Set(path []interface{}, value interface{}) TreeBranch { return set(branch, path, value).(TreeBranch) } +func unset(branch interface{}, path []interface{}) (interface{}, error) { + switch branch := branch.(type) { + case TreeBranch: + for i, item := range branch { + if item.Key == path[0] { + if len(path) == 1 { + branch = slices.Delete(branch, i, i+1) + } else { + v, err := unset(item.Value, path[1:]) + if err != nil { + return nil, err + } + branch[i].Value = v + } + return branch, nil + } + } + return nil, &SopsKeyNotFound{Msg: "Key not found: %s", Key: path[0]} + case []interface{}: + position := path[0].(int) + if position >= len(branch) { + return nil, &SopsKeyNotFound{Msg: "Index %d out of bounds", Key: path[0]} + } + if len(path) == 1 { + branch = slices.Delete(branch, position, position+1) + } else { + v, err := unset(branch[position], path[1:]) + if err != nil { + return nil, err + } + branch[position] = v + } + return branch, nil + default: + return nil, fmt.Errorf("Unsupported type: %T for item '%s'", branch, path[0]) + } +} + +// Unset removes a value on a given tree from the specified path +func (branch TreeBranch) Unset(path []interface{}) (TreeBranch, error) { + v, err := unset(branch, path) + if err != nil { + return nil, err + } + return v.(TreeBranch), nil +} + // Tree is the data structure used by sops to represent documents internally type Tree struct { Metadata Metadata