diff --git a/README.rst b/README.rst index 239a5ac4b..2b6280f8b 100644 --- a/README.rst +++ b/README.rst @@ -791,6 +791,31 @@ By default ``sops`` just dumps all the output to the standard output. We can use Beware using both ``--in-place`` and ``--output`` flags will result in an error. +Using the publish command +~~~~~~~~~~~~~~~~~~~~~~~~~ +``sops publish $file`` publishes a file to a pre-configured destination (this lives in the sops +config file). Additionally, support re-encryption rules that work just like the creation rules. + +This command requires a ``.sops.yaml`` configuration file. Below is an example: + +.. code:: yaml + + destination_rules: + - s3_bucket: "sops-secrets" + path_regex: s3/* + recreation_rule: + pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307 + - gcs_bucket: "sops-secrets" + path_regex: gcs/* + recreation_rule: + pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307 + +The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets`` and +will place all files under ``gcs/*`` into the GCS bucket ``sops-secrets``. As well, it will decrypt +these files and re-encrypt them using the ``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key. + +You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` + Important information on types ------------------------------ diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 56501c855..d9b819b77 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -8,9 +8,11 @@ import ( "strings" "time" + "github.com/fatih/color" wordwrap "github.com/mitchellh/go-wordwrap" "go.mozilla.org/sops" "go.mozilla.org/sops/cmd/sops/codes" + "go.mozilla.org/sops/keys" "go.mozilla.org/sops/keyservice" "go.mozilla.org/sops/kms" "go.mozilla.org/sops/stores/dotenv" @@ -339,3 +341,65 @@ func RecoverDataKeyFromBuggyKMS(opts GenericDecryptOpts, tree *sops.Tree) []byte return nil } + +type Diff struct { + Common []keys.MasterKey + Added []keys.MasterKey + Removed []keys.MasterKey +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func DiffKeyGroups(ours, theirs []sops.KeyGroup) []Diff { + var diffs []Diff + for i := 0; i < max(len(ours), len(theirs)); i++ { + var diff Diff + var ourGroup, theirGroup sops.KeyGroup + if len(ours) > i { + ourGroup = ours[i] + } + if len(theirs) > i { + theirGroup = theirs[i] + } + ourKeys := make(map[string]struct{}) + theirKeys := make(map[string]struct{}) + for _, key := range ourGroup { + ourKeys[key.ToString()] = struct{}{} + } + for _, key := range theirGroup { + if _, ok := ourKeys[key.ToString()]; ok { + diff.Common = append(diff.Common, key) + } else { + diff.Added = append(diff.Added, key) + } + theirKeys[key.ToString()] = struct{}{} + } + for _, key := range ourGroup { + if _, ok := theirKeys[key.ToString()]; !ok { + diff.Removed = append(diff.Removed, key) + } + } + diffs = append(diffs, diff) + } + return diffs +} + +func PrettyPrintDiffs(diffs []Diff) { + for i, diff := range diffs { + color.New(color.Underline).Printf("Group %d\n", i+1) + for _, c := range diff.Common { + fmt.Printf(" %s\n", c.ToString()) + } + for _, c := range diff.Added { + color.New(color.FgGreen).Printf("+++ %s\n", c.ToString()) + } + for _, c := range diff.Removed { + color.New(color.FgRed).Printf("--- %s\n", c.ToString()) + } + } +} diff --git a/cmd/sops/main.go b/cmd/sops/main.go index edaa6e49e..34d67f4e1 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -21,6 +21,7 @@ import ( "go.mozilla.org/sops/cmd/sops/common" "go.mozilla.org/sops/cmd/sops/subcommand/groups" keyservicecmd "go.mozilla.org/sops/cmd/sops/subcommand/keyservice" + publishcmd "go.mozilla.org/sops/cmd/sops/subcommand/publish" "go.mozilla.org/sops/cmd/sops/subcommand/updatekeys" "go.mozilla.org/sops/config" "go.mozilla.org/sops/gcpkms" @@ -105,6 +106,49 @@ func main() { For more information, see the README at github.com/mozilla/sops` app.EnableBashCompletion = true app.Commands = []cli.Command{ + { + Name: "publish", + Usage: "Publish sops file to a configured destination", + ArgsUsage: `file`, + Flags: append([]cli.Flag{ + cli.BoolFlag{ + Name: "yes, y", + Usage: `pre-approve all changes and run non-interactively`, + }, + cli.BoolFlag{ + Name: "verbose", + Usage: "Enable verbose logging output", + }, + }, keyserviceFlags...), + Action: func(c *cli.Context) error { + if c.Bool("verbose") || c.GlobalBool("verbose") { + logging.SetLevel(logrus.DebugLevel) + } + configPath, err := config.FindConfigFile(".") + if err != nil { + return common.NewExitError(err, codes.ErrorGeneric) + } + if c.NArg() < 1 { + return common.NewExitError("Error: no file specified", codes.NoFileSpecified) + } + fileName := c.Args()[0] + inputStore := inputStore(c, fileName) + err = publishcmd.Run(publishcmd.Opts{ + ConfigPath: configPath, + InputPath: fileName, + InputStore: inputStore, + Cipher: aes.NewCipher(), + KeyServices: keyservices(c), + Interactive: !c.Bool("yes"), + }) + if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { + return cliErr + } else if err != nil { + return common.NewExitError(err, codes.ErrorGeneric) + } + return nil + }, + }, { Name: "keyservice", Usage: "start a SOPS key service server", @@ -129,7 +173,7 @@ func main() { }, }, Action: func(c *cli.Context) error { - if c.Bool("verbose") { + if c.Bool("verbose") || c.GlobalBool("verbose") { logging.SetLevel(logrus.DebugLevel) } err := keyservicecmd.Run(keyservicecmd.Opts{ diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go new file mode 100644 index 000000000..fc4cb5afe --- /dev/null +++ b/cmd/sops/subcommand/publish/publish.go @@ -0,0 +1,151 @@ +package publish + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "go.mozilla.org/sops" + "go.mozilla.org/sops/cmd/sops/codes" + "go.mozilla.org/sops/cmd/sops/common" + "go.mozilla.org/sops/config" + "go.mozilla.org/sops/keyservice" + "go.mozilla.org/sops/logging" + "go.mozilla.org/sops/version" + + "github.com/sirupsen/logrus" +) + +var log *logrus.Logger + +func init() { + log = logging.NewLogger("PUBLISH") +} + +type Opts struct { + Interactive bool + Cipher sops.Cipher + ConfigPath string + InputPath string + KeyServices []keyservice.KeyServiceClient + InputStore sops.Store +} + +func Run(opts Opts) error { + var fileContents []byte + path, err := filepath.Abs(opts.InputPath) + if err != nil { + return err + } + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() { + return fmt.Errorf("can't operate on a directory") + } + _, fileName := filepath.Split(path) + + conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string)) + if err != nil { + return err + } + if conf.Destination == nil { + return errors.New("no destination configured for this file") + } + + // Check that this is a sops-encrypted file + tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath) + if err != nil { + return err + } + + // Re-encrypt if settings exist to do so + if len(conf.KeyGroups[0]) != 0 { + log.Debug("Re-encrypting tree before publishing") + _, err = common.DecryptTree(common.DecryptTreeOpts{ + Cipher: opts.Cipher, + IgnoreMac: false, + Tree: tree, + KeyServices: opts.KeyServices, + }) + if err != nil { + return err + } + + diffs := common.DiffKeyGroups(tree.Metadata.KeyGroups, conf.KeyGroups) + keysWillChange := false + for _, diff := range diffs { + if len(diff.Added) > 0 || len(diff.Removed) > 0 { + keysWillChange = true + } + } + if keysWillChange { + fmt.Printf("The following changes will be made to the file's key groups:\n") + common.PrettyPrintDiffs(diffs) + } + + tree.Metadata = sops.Metadata{ + KeyGroups: conf.KeyGroups, + UnencryptedSuffix: conf.UnencryptedSuffix, + EncryptedSuffix: conf.EncryptedSuffix, + Version: version.Version, + ShamirThreshold: conf.ShamirThreshold, + } + + dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices) + if len(errs) > 0 { + err = fmt.Errorf("Could not generate data key: %s", errs) + return err + } + + err = common.EncryptTree(common.EncryptTreeOpts{ + DataKey: dataKey, + Tree: tree, + Cipher: opts.Cipher, + }) + if err != nil { + return err + } + + fileContents, err = opts.InputStore.EmitEncryptedFile(*tree) + if err != nil { + return common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree) + } + } else { + fileContents, err = ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("could not read file: %s", err) + } + } + + if opts.Interactive { + var response string + for response != "y" && response != "n" { + fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName)) + _, err := fmt.Scanln(&response) + if err != nil { + return err + } + } + if response == "n" { + return errors.New("Publish canceled") + } + } + + err = conf.Destination.Upload(fileContents, fileName) + if err != nil { + return err + } + + return nil +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/cmd/sops/subcommand/updatekeys/updatekeys.go b/cmd/sops/subcommand/updatekeys/updatekeys.go index d63a0d93e..2f403770e 100644 --- a/cmd/sops/subcommand/updatekeys/updatekeys.go +++ b/cmd/sops/subcommand/updatekeys/updatekeys.go @@ -6,11 +6,9 @@ import ( "os" "path/filepath" - "github.com/fatih/color" - "go.mozilla.org/sops" + "go.mozilla.org/sops/cmd/sops/codes" "go.mozilla.org/sops/cmd/sops/common" "go.mozilla.org/sops/config" - "go.mozilla.org/sops/keys" "go.mozilla.org/sops/keyservice" ) @@ -48,10 +46,11 @@ func updateFile(opts Opts) error { if err != nil { return err } - diffs := diffKeyGroups(tree.Metadata.KeyGroups, conf.KeyGroups) + + diffs := common.DiffKeyGroups(tree.Metadata.KeyGroups, conf.KeyGroups) keysWillChange := false for _, diff := range diffs { - if len(diff.added) > 0 || len(diff.removed) > 0 { + if len(diff.Added) > 0 || len(diff.Removed) > 0 { keysWillChange = true } } @@ -60,18 +59,8 @@ func updateFile(opts Opts) error { return nil } fmt.Printf("The following changes will be made to the file's groups:\n") - for i, diff := range diffs { - color.New(color.Underline).Printf("Group %d\n", i+1) - for _, c := range diff.common { - fmt.Printf(" %s\n", c.ToString()) - } - for _, c := range diff.added { - color.New(color.FgGreen).Printf("+++ %s\n", c.ToString()) - } - for _, c := range diff.removed { - color.New(color.FgRed).Printf("--- %s\n", c.ToString()) - } - } + common.PrettyPrintDiffs(diffs) + if opts.Interactive { var response string for response != "y" && response != "n" { @@ -88,7 +77,7 @@ func updateFile(opts Opts) error { } key, err := tree.Metadata.GetDataKeyWithKeyServices(opts.KeyServices) if err != nil { - return fmt.Errorf("error getting data key: %s", err) + return common.NewExitError(err, codes.CouldNotRetrieveKey) } tree.Metadata.KeyGroups = conf.KeyGroups if opts.GroupQuorum != 0 { @@ -101,7 +90,7 @@ func updateFile(opts Opts) error { } output, err := store.EmitEncryptedFile(*tree) if err != nil { - return fmt.Errorf("error marshaling tree: %s", err) + return common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree) } outputFile, err := os.Create(opts.InputPath) if err != nil { @@ -116,56 +105,9 @@ func updateFile(opts Opts) error { return nil } -type diff struct { - common []keys.MasterKey - added []keys.MasterKey - removed []keys.MasterKey -} - -func max(a, b int) int { - if a > b { - return a - } - return b -} - func min(a, b int) int { if a < b { return a } return b } - -func diffKeyGroups(ours, theirs []sops.KeyGroup) []diff { - var diffs []diff - for i := 0; i < max(len(ours), len(theirs)); i++ { - var diff diff - var ourGroup, theirGroup sops.KeyGroup - if len(ours) > i { - ourGroup = ours[i] - } - if len(theirs) > i { - theirGroup = theirs[i] - } - ourKeys := make(map[string]struct{}) - theirKeys := make(map[string]struct{}) - for _, key := range ourGroup { - ourKeys[key.ToString()] = struct{}{} - } - for _, key := range theirGroup { - if _, ok := ourKeys[key.ToString()]; ok { - diff.common = append(diff.common, key) - } else { - diff.added = append(diff.added, key) - } - theirKeys[key.ToString()] = struct{}{} - } - for _, key := range ourGroup { - if _, ok := theirKeys[key.ToString()]; !ok { - diff.removed = append(diff.removed, key) - } - } - diffs = append(diffs, diff) - } - return diffs -} diff --git a/config/config.go b/config/config.go index 07628ef0f..ea5144698 100644 --- a/config/config.go +++ b/config/config.go @@ -18,6 +18,7 @@ import ( "go.mozilla.org/sops/kms" "go.mozilla.org/sops/logging" "go.mozilla.org/sops/pgp" + "go.mozilla.org/sops/publish" ) var log *logrus.Logger @@ -60,7 +61,8 @@ func FindConfigFile(start string) (string, error) { } type configFile struct { - CreationRules []creationRule `yaml:"creation_rules"` + CreationRules []creationRule `yaml:"creation_rules"` + DestinationRules []destinationRule `yaml:"destination_rules"` } type keyGroup struct { @@ -87,8 +89,16 @@ type azureKVKey struct { Version string `yaml:"version"` } +type destinationRule struct { + PathRegex string `yaml:"path_regex"` + S3Bucket string `yaml:"s3_bucket"` + S3Prefix string `yaml:"s3_prefix"` + GCSBucket string `yaml:"gcs_bucket"` + GCSPrefix string `yaml:"gcs_prefix"` + RecreationRule creationRule `yaml:"recreation_rule,omitempty"` +} + type creationRule struct { - FilenameRegex string `yaml:"filename_regex"` PathRegex string `yaml:"path_regex"` KMS string AwsProfile string `yaml:"aws_profile"` @@ -116,50 +126,13 @@ type Config struct { ShamirThreshold int UnencryptedSuffix string EncryptedSuffix string + Destination publish.Destination } -func loadForFileFromBytes(confBytes []byte, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { - conf := configFile{} - err := conf.load(confBytes) - if err != nil { - return nil, fmt.Errorf("error loading config: %s", err) - } - var rule *creationRule - - for _, r := range conf.CreationRules { - if r.PathRegex == "" && r.FilenameRegex == "" { - rule = &r - break - } - if r.PathRegex != "" && r.FilenameRegex != "" { - return nil, fmt.Errorf("error loading config: both filename_regex and path_regex were found, use only path_regex") - } - if r.FilenameRegex != "" { - if match, _ := regexp.MatchString(r.FilenameRegex, filePath); match { - log.Warn("The key: filename_regex will be removed in a future release. Instead use key: path_regex in your .sops.yaml file") - rule = &r - break - } - } - if r.PathRegex != "" { - if match, _ := regexp.MatchString(r.PathRegex, filePath); match { - rule = &r - break - } - } - } - - if rule == nil { - return nil, fmt.Errorf("error loading config: no matching creation rules found") - } - - if rule.UnencryptedSuffix != "" && rule.EncryptedSuffix != "" { - return nil, fmt.Errorf("error loading config: cannot use both encrypted_suffix and unencrypted_suffix for the same rule") - } - +func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) { var groups []sops.KeyGroup - if len(rule.KeyGroups) > 0 { - for _, group := range rule.KeyGroups { + if len(cRule.KeyGroups) > 0 { + for _, group := range cRule.KeyGroups { var keyGroup sops.KeyGroup for _, k := range group.PGP { keyGroup = append(keyGroup, pgp.NewMasterKeyFromFingerprint(k)) @@ -174,16 +147,16 @@ func loadForFileFromBytes(confBytes []byte, filePath string, kmsEncryptionContex } } else { var keyGroup sops.KeyGroup - for _, k := range pgp.MasterKeysFromFingerprintString(rule.PGP) { + for _, k := range pgp.MasterKeysFromFingerprintString(cRule.PGP) { keyGroup = append(keyGroup, k) } - for _, k := range kms.MasterKeysFromArnString(rule.KMS, kmsEncryptionContext, rule.AwsProfile) { + for _, k := range kms.MasterKeysFromArnString(cRule.KMS, kmsEncryptionContext, cRule.AwsProfile) { keyGroup = append(keyGroup, k) } - for _, k := range gcpkms.MasterKeysFromResourceIDString(rule.GCPKMS) { + for _, k := range gcpkms.MasterKeysFromResourceIDString(cRule.GCPKMS) { keyGroup = append(keyGroup, k) } - azureKeys, err := azkv.MasterKeysFromURLs(rule.AzureKeyVault) + azureKeys, err := azkv.MasterKeysFromURLs(cRule.AzureKeyVault) if err != nil { return nil, err } @@ -192,6 +165,32 @@ func loadForFileFromBytes(confBytes []byte, filePath string, kmsEncryptionContex } groups = append(groups, keyGroup) } + return groups, nil +} + +func loadConfigFile(confPath string) (*configFile, error) { + confBytes, err := ioutil.ReadFile(confPath) + if err != nil { + return nil, fmt.Errorf("could not read config file: %s", err) + } + conf := &configFile{} + err = conf.load(confBytes) + if err != nil { + return nil, fmt.Errorf("error loading config: %s", err) + } + return conf, nil +} + +func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string) (*Config, error) { + if rule.UnencryptedSuffix != "" && rule.EncryptedSuffix != "" { + return nil, fmt.Errorf("error loading config: cannot use both encrypted_suffix and unencrypted_suffix for the same rule") + } + + groups, err := getKeyGroupsFromCreationRule(rule, kmsEncryptionContext) + if err != nil { + return nil, err + } + return &Config{ KeyGroups: groups, ShamirThreshold: rule.ShamirThreshold, @@ -200,13 +199,98 @@ func loadForFileFromBytes(confBytes []byte, filePath string, kmsEncryptionContex }, nil } +func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { + var rule *creationRule + var dRule *destinationRule + + if len(conf.DestinationRules) > 0 { + for _, r := range conf.DestinationRules { + if r.PathRegex == "" { + dRule = &r + rule = &dRule.RecreationRule + break + } + if r.PathRegex != "" { + if match, _ := regexp.MatchString(r.PathRegex, filePath); match { + dRule = &r + rule = &dRule.RecreationRule + break + } + } + } + } + + if dRule == nil { + return nil, fmt.Errorf("error loading config: no matching destination found in config") + } + + var dest publish.Destination + if dRule != nil { + if dRule.S3Bucket != "" && dRule.GCSBucket != "" { + return nil, fmt.Errorf("error loading config: both s3_bucket and gcs_bucket were found for destination rule, you can only use one.") + } + if dRule.S3Bucket != "" { + dest = publish.NewS3Destination(dRule.S3Bucket, dRule.S3Prefix) + } + if dRule.GCSBucket != "" { + dest = publish.NewGCSDestination(dRule.GCSBucket, dRule.GCSPrefix) + } + } + + config, err := configFromRule(rule, kmsEncryptionContext) + if err != nil { + return nil, err + } + config.Destination = dest + + return config, nil +} + +func parseCreationRuleForFile(conf *configFile, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { + var rule *creationRule + + for _, r := range conf.CreationRules { + if r.PathRegex == "" { + rule = &r + break + } + if r.PathRegex != "" { + if match, _ := regexp.MatchString(r.PathRegex, filePath); match { + rule = &r + break + } + } + } + + if rule == nil { + return nil, fmt.Errorf("error loading config: no matching creation rules found") + } + + config, err := configFromRule(rule, kmsEncryptionContext) + if err != nil { + return nil, err + } + + return config, nil +} + // LoadForFile load the configuration for a given SOPS file from the config file at confPath. A kmsEncryptionContext // should be provided for configurations that do not contain key groups, as there's no way to specify context inside // a SOPS config file outside of key groups. func LoadForFile(confPath string, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { - confBytes, err := ioutil.ReadFile(confPath) + conf, err := loadConfigFile(confPath) if err != nil { - return nil, fmt.Errorf("could not read config file: %s", err) + return nil, err } - return loadForFileFromBytes(confBytes, filePath, kmsEncryptionContext) + return parseCreationRuleForFile(conf, filePath, kmsEncryptionContext) +} + +// LoadDestinationRuleForFile works the same as LoadForFile, but gets the "creation_rule" from the matching destination_rule's +// "recreation_rule". +func LoadDestinationRuleForFile(confPath string, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) { + conf, err := loadConfigFile(confPath) + if err != nil { + return nil, err + } + return parseDestinationRuleForFile(conf, filePath, kmsEncryptionContext) } diff --git a/config/config_test.go b/config/config_test.go index 5db731c4d..a5bda301b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -25,7 +25,7 @@ func TestFindConfigFileRecursive(t *testing.T) { return nil, &os.PathError{} }} filepath, err := FindConfigFile(".") - assert.Equal(t, nil, err) + assert.Nil(t, err) assert.Equal(t, expectedPath, filepath) } @@ -38,7 +38,7 @@ func TestFindConfigFileCurrentDir(t *testing.T) { return nil, &os.PathError{} }} filepath, err := FindConfigFile(".") - assert.Equal(t, nil, err) + assert.Nil(t, err) assert.Equal(t, expectedPath, filepath) } @@ -60,7 +60,7 @@ creation_rules: kms: "1" pgp: "2" gcp_kms: "3" - - filename_regex: "somefilename.yml" + - path_regex: somefilename.yml kms: bilbo pgp: baggins gcp_kms: precious @@ -123,16 +123,41 @@ var sampleInvalidConfig = []byte(` creation_rules: `) +var sampleConfigWithDestinationRule = []byte(` +creation_rules: + - path_regex: foobar* + kms: "1" + pgp: "2" + gcp_kms: "3" + - path_regex: "" + kms: foo + pgp: bar + gcp_kms: baz +destination_rules: + - path_regex: "" + s3_bucket: "foobar" + s3_prefix: "test/" + recreation_rule: + pgp: newpgp +`) + +func parseConfigFile(confBytes []byte, t *testing.T) *configFile { + conf := &configFile{} + err := conf.load(confBytes) + assert.Nil(t, err) + return conf +} + func TestLoadConfigFile(t *testing.T) { expected := configFile{ CreationRules: []creationRule{ - creationRule{ + { PathRegex: "foobar*", KMS: "1", PGP: "2", GCPKMS: "3", }, - creationRule{ + { PathRegex: "", KMS: "foo", PGP: "bar", @@ -143,7 +168,7 @@ func TestLoadConfigFile(t *testing.T) { conf := configFile{} err := conf.load(sampleConfig) - assert.Equal(t, nil, err) + assert.Nil(t, err) assert.Equal(t, expected, conf) } @@ -178,44 +203,44 @@ func TestLoadConfigFileWithGroups(t *testing.T) { conf := configFile{} err := conf.load(sampleConfigWithGroups) - assert.Equal(t, nil, err) + assert.Nil(t, err) assert.Equal(t, expected, conf) } func TestLoadInvalidConfigFile(t *testing.T) { - _, err := loadForFileFromBytes(sampleInvalidConfig, "foobar2000", nil) + _, err := parseCreationRuleForFile(parseConfigFile(sampleInvalidConfig, t), "foobar2000", nil) assert.NotNil(t, err) } func TestKeyGroupsForFile(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfig, "foobar2000", nil) - assert.Equal(t, nil, err) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "foobar2000", nil) + assert.Nil(t, err) assert.Equal(t, "2", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "1", conf.KeyGroups[0][1].ToString()) - conf, err = loadForFileFromBytes(sampleConfig, "whatever", nil) - assert.Equal(t, nil, err) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil) + assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) } func TestKeyGroupsForFileWithPath(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfigWithPath, "foo/bar2000", nil) - assert.Equal(t, nil, err) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "foo/bar2000", nil) + assert.Nil(t, err) assert.Equal(t, "2", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "1", conf.KeyGroups[0][1].ToString()) - conf, err = loadForFileFromBytes(sampleConfigWithPath, "somefilename.yml", nil) - assert.Equal(t, nil, err) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "somefilename.yml", nil) + assert.Nil(t, err) assert.Equal(t, "baggins", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "bilbo", conf.KeyGroups[0][1].ToString()) - conf, err = loadForFileFromBytes(sampleConfig, "whatever", nil) - assert.Equal(t, nil, err) + conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil) + assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) } func TestKeyGroupsForFileWithGroups(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfigWithGroups, "whatever", nil) - assert.Equal(t, nil, err) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithGroups, t), "whatever", nil) + assert.Nil(t, err) assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString()) assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString()) assert.Equal(t, "qux", conf.KeyGroups[1][0].ToString()) @@ -223,18 +248,26 @@ func TestKeyGroupsForFileWithGroups(t *testing.T) { } func TestLoadConfigFileWithUnencryptedSuffix(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfigWithSuffixParameters, "foobar", nil) - assert.Equal(t, nil, err) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "foobar", nil) + assert.Nil(t, err) assert.Equal(t, "_unencrypted", conf.UnencryptedSuffix) } func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) { - conf, err := loadForFileFromBytes(sampleConfigWithSuffixParameters, "barfoo", nil) - assert.Equal(t, nil, err) + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "barfoo", nil) + assert.Nil(t, err) assert.Equal(t, "_enc", conf.EncryptedSuffix) } func TestLoadConfigFileWithInvalidParameters(t *testing.T) { - _, err := loadForFileFromBytes(sampleConfigWithInvalidParameters, "foobar", nil) + _, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "foobar", nil) assert.NotNil(t, err) } + +func TestLoadConfigFileWithDestinationRule(t *testing.T) { + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithDestinationRule, t), "barfoo", nil) + assert.Nil(t, err) + assert.Equal(t, "newpgp", conf.KeyGroups[0][0].ToString()) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "s3://foobar/test/barfoo", conf.Destination.Path("barfoo")) +} diff --git a/functional-tests/.sops.yaml b/functional-tests/.sops.yaml index 6280ff3d7..b5ce5f16a 100644 --- a/functional-tests/.sops.yaml +++ b/functional-tests/.sops.yaml @@ -1,14 +1,20 @@ creation_rules: - - filename_regex: test_roundtrip_keygroups.yaml + - path_regex: test_roundtrip_keygroups.yaml key_groups: - pgp: - 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A - pgp: - 729D26A79482B5A20DEAD0A76945978B930DD7A2 - - filename_regex: test_roundtrip_keygroups_missing_decryption_key.yaml + - path_regex: test_roundtrip_keygroups_missing_decryption_key.yaml key_groups: - pgp: - 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A - pgp: - 620B9A4C96230B91E7473D20113D2B26EA0890C7 - pgp: 1022470DE3F0BC54BC6AB62DE05550BC07FB1A0A +destination_rules: + - s3_bucket: "sops-publish-functional-tests" + s3_prefix: "functional-test/" + path_regex: test_encrypt_publish_s3.json + reencryption_rule: + pgp: 620B9A4C96230B91E7473D20113D2B26EA0890C7 diff --git a/functional-tests/src/lib.rs b/functional-tests/src/lib.rs index 3e00ad4c7..f0712d12e 100644 --- a/functional-tests/src/lib.rs +++ b/functional-tests/src/lib.rs @@ -74,6 +74,37 @@ mod tests { } } + #[test] + #[ignore] + fn publish_json_file_s3() { + let file_path = prepare_temp_file("test_encrypt_publish_s3.json", + b"{ + \"foo\": 2, + \"bar\": \"baz\" +}"); + assert!(Command::new(SOPS_BINARY_PATH) + .arg("-e") + .arg("-i") + .arg(file_path.clone()) + .output() + .expect("Error running sops") + .status + .success(), + "SOPS failed to encrypt a file"); + assert!(Command::new(SOPS_BINARY_PATH) + .arg("publish") + .arg("--yes") + .arg(file_path.clone()) + .output() + .expect("Error running sops") + .status + .success(), + "sops failed to publish a file to S3"); + + //TODO: Check that file exists in S3 Bucket + } + + #[test] #[ignore] fn encrypt_json_file_kms() { @@ -260,7 +291,7 @@ b: ba"# } panic!("Output YAML does not have the expected structure"); } - + #[test] fn set_yaml_file_string() { let file_path = prepare_temp_file("test_set_string.yaml", diff --git a/publish/gcs.go b/publish/gcs.go new file mode 100644 index 000000000..de5e574d5 --- /dev/null +++ b/publish/gcs.go @@ -0,0 +1,36 @@ +package publish + +import ( + "context" + "fmt" + + "cloud.google.com/go/storage" +) + +type GCSDestination struct { + gcsBucket string + gcsPrefix string +} + +func NewGCSDestination(gcsBucket string, gcsPrefix string) *GCSDestination { + return &GCSDestination{gcsBucket, gcsPrefix} +} + +func (gcsd *GCSDestination) Path(fileName string) string { + return fmt.Sprintf("gcs://%s/%s%s", gcsd.gcsBucket, gcsd.gcsPrefix, fileName) +} + +func (gcsd *GCSDestination) Upload(fileContents []byte, fileName string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + return err + } + wc := client.Bucket(gcsd.gcsBucket).Object(gcsd.gcsPrefix + fileName).NewWriter(ctx) + defer wc.Close() + _, err = wc.Write(fileContents) + if err != nil { + return err + } + return nil +} diff --git a/publish/publish.go b/publish/publish.go new file mode 100644 index 000000000..9a559858c --- /dev/null +++ b/publish/publish.go @@ -0,0 +1,6 @@ +package publish + +type Destination interface { + Upload(fileContents []byte, fileName string) error + Path(fileName string) string +} diff --git a/publish/s3.go b/publish/s3.go new file mode 100644 index 000000000..65ca6ab5d --- /dev/null +++ b/publish/s3.go @@ -0,0 +1,38 @@ +package publish + +import ( + "bytes" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +type S3Destination struct { + s3Bucket string + s3Prefix string +} + +func NewS3Destination(s3Bucket string, s3Prefix string) *S3Destination { + return &S3Destination{s3Bucket, s3Prefix} +} + +func (s3d *S3Destination) Path(fileName string) string { + return fmt.Sprintf("s3://%s/%s%s", s3d.s3Bucket, s3d.s3Prefix, fileName) +} + +func (s3d *S3Destination) Upload(fileContents []byte, fileName string) error { + sess := session.Must(session.NewSession()) + svc := s3.New(sess) + input := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(bytes.NewReader(fileContents)), + Bucket: aws.String(s3d.s3Bucket), + Key: aws.String(s3d.s3Prefix + fileName), + } + _, err := svc.PutObject(input) + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/googleapis/gax-go/CODE_OF_CONDUCT.md b/vendor/github.com/googleapis/gax-go/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..46b2a08ea --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/CODE_OF_CONDUCT.md @@ -0,0 +1,43 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, +and in the interest of fostering an open and welcoming community, +we pledge to respect all people who contribute through reporting issues, +posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project +a harassment-free experience for everyone, +regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, +such as physical or electronic +addresses, without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct. +By adopting this Code of Conduct, +project maintainers commit themselves to fairly and consistently +applying these principles to every aspect of managing this project. +Project maintainers who do not follow or enforce the Code of Conduct +may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by opening an issue +or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.2.0, +available at [http://contributor-covenant.org/version/1/2/0/](http://contributor-covenant.org/version/1/2/0/) diff --git a/vendor/github.com/googleapis/gax-go/CONTRIBUTING.md b/vendor/github.com/googleapis/gax-go/CONTRIBUTING.md new file mode 100644 index 000000000..b0404fb64 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/CONTRIBUTING.md @@ -0,0 +1,32 @@ +Want to contribute? Great! First, read this page (including the small print at the end). + +### Before you contribute +Before we can use your code, you must sign the +[Google Individual Contributor License Agreement] +(https://cla.developers.google.com/about/google-individual) +(CLA), which you can do online. The CLA is necessary mainly because you own the +copyright to your changes, even after your contribution becomes part of our +codebase, so we need your permission to use and distribute your code. We also +need to be sure of various other things—for instance that you'll tell us if you +know that your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. +Before you start working on a larger contribution, you should get in touch with +us first through the issue tracker with your idea so that we can help out and +possibly guide you. Coordinating up front makes it much easier to avoid +frustration later on. + +### Code reviews +All submissions, including submissions by project members, require review. We +use Github pull requests for this purpose. + +### Breaking code changes +When a breaking change is added, CI/CD will fail. If the change is expected, +add a BREAKING_CHANGE_ACCEPTABLE= line to the CL description. This will +cause CI/CD to skip checking breaking changes. + +### The small print +Contributions made by corporations are covered by a different agreement than +the one above, the +[Software Grant and Corporate Contributor License Agreement] +(https://cla.developers.google.com/about/google-corporate). diff --git a/vendor/github.com/googleapis/gax-go/LICENSE b/vendor/github.com/googleapis/gax-go/LICENSE new file mode 100644 index 000000000..6d16b6578 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/LICENSE @@ -0,0 +1,27 @@ +Copyright 2016, Google Inc. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/googleapis/gax-go/README.md b/vendor/github.com/googleapis/gax-go/README.md new file mode 100644 index 000000000..c3bb2e187 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/README.md @@ -0,0 +1,27 @@ +Google API Extensions for Go +============================ + +[![GoDoc](https://godoc.org/github.com/googleapis/gax-go?status.svg)](https://godoc.org/github.com/googleapis/gax-go) + +Google API Extensions for Go (gax-go) is a set of modules which aids the +development of APIs for clients and servers based on `gRPC` and Google API +conventions. + +To install the API extensions, use: + +``` +go get -u github.com/googleapis/gax-go +``` + +**Note:** Application code will rarely need to use this library directly, +but the code generated automatically from API definition files can use it +to simplify code generation and to provide more convenient and idiomatic API surface. + +Go Versions +=========== +This library requires Go 1.6 or above. + +License +======= +BSD - please see [LICENSE](https://github.com/googleapis/gax-go/blob/master/LICENSE) +for more information. diff --git a/vendor/github.com/googleapis/gax-go/RELEASING.md b/vendor/github.com/googleapis/gax-go/RELEASING.md new file mode 100644 index 000000000..0557654b2 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/RELEASING.md @@ -0,0 +1,30 @@ +# How to release v1 + +1. Determine the current release version with `git tag -l`. It should look + something like `vX.Y.Z`. We'll call the current version `$CV` and the new + version `$NV`. +1. On master, run `git log $CV..` to list all the changes since the last + release. + a. NOTE: Some commits may pertain to only v1 or v2. Manually introspect + each commit to figure which occurred in v1. +1. Edit `CHANGES.md` to include a summary of the changes. +1. Mail the CL containing the `CHANGES.md` changes. When the CL is approved, + submit it. +1. Without submitting any other CLs: + a. Switch to master. + b. `git pull` + c. Tag the repo with the next version: `git tag $NV`. It should be of the + form `v1.Y.Z`. + d. Push the tag: `git push origin $NV`. +1. Update [the releases page](https://github.com/googleapis/google-cloud-go/releases) + with the new release, copying the contents of the CHANGES.md. + +# How to release v2 + +Same process as v1, once again noting that the commit list may include v1 +commits (which should be pruned out). Note also whilst v1 tags are `v1.Y.Z`, v2 +tags are `v2.Y.Z`. + +# On releasing multiple major versions + +Please see https://github.com/golang/go/wiki/Modules#releasing-modules-v2-or-higher. diff --git a/vendor/github.com/googleapis/gax-go/call_option.go b/vendor/github.com/googleapis/gax-go/call_option.go new file mode 100644 index 000000000..5bd48972b --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/call_option.go @@ -0,0 +1,71 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + v2 "github.com/googleapis/gax-go/v2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +// CallOption is an option used by Invoke to control behaviors of RPC calls. +// CallOption works by modifying relevant fields of CallSettings. +type CallOption = v2.CallOption + +// Retryer is used by Invoke to determine retry behavior. +type Retryer = v2.Retryer + +// WithRetry sets CallSettings.Retry to fn. +func WithRetry(fn func() Retryer) CallOption { + return v2.WithRetry(fn) +} + +// OnCodes returns a Retryer that retries if and only if +// the previous attempt returns a GRPC error whose error code is stored in cc. +// Pause times between retries are specified by bo. +// +// bo is only used for its parameters; each Retryer has its own copy. +func OnCodes(cc []codes.Code, bo Backoff) Retryer { + return v2.OnCodes(cc, bo) +} + +// Backoff implements exponential backoff. +// The wait time between retries is a random value between 0 and the "retry envelope". +// The envelope starts at Initial and increases by the factor of Multiplier every retry, +// but is capped at Max. +type Backoff = v2.Backoff + +// WithGRPCOptions allows passing gRPC call options during client creation. +func WithGRPCOptions(opt ...grpc.CallOption) CallOption { + return v2.WithGRPCOptions(opt...) +} + +// CallSettings allow fine-grained control over how calls are made. +type CallSettings = v2.CallSettings diff --git a/vendor/github.com/googleapis/gax-go/gax.go b/vendor/github.com/googleapis/gax-go/gax.go new file mode 100644 index 000000000..006739804 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/gax.go @@ -0,0 +1,39 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Package gax contains a set of modules which aid the development of APIs +// for clients and servers based on gRPC and Google API conventions. +// +// Application code will rarely need to use this library directly. +// However, code generated automatically from API definition files can use it +// to simplify code generation and to provide more convenient and idiomatic API surfaces. +package gax + +// Version specifies the gax version. +const Version = "1.0.1" diff --git a/vendor/github.com/googleapis/gax-go/go.mod b/vendor/github.com/googleapis/gax-go/go.mod new file mode 100644 index 000000000..acfd6c9c6 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/go.mod @@ -0,0 +1,11 @@ +module github.com/googleapis/gax-go + +require ( + github.com/golang/protobuf v1.3.1 + github.com/googleapis/gax-go/v2 v2.0.2 + golang.org/x/exp v0.0.0-20190221220918-438050ddec5e + golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3 + golang.org/x/tools v0.0.0-20190114222345-bf090417da8b + google.golang.org/grpc v1.19.0 + honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 +) diff --git a/vendor/github.com/googleapis/gax-go/go.sum b/vendor/github.com/googleapis/gax-go/go.sum new file mode 100644 index 000000000..2c11f2f2c --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/go.sum @@ -0,0 +1,44 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/googleapis/gax-go/v2 v2.0.2 h1:/rNgUniLy2vDXiK2xyJOcirGpC3G99dtK1NWx26WZ8Y= +github.com/googleapis/gax-go/v2 v2.0.2/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +golang.org/x/exp v0.0.0-20190221220918-438050ddec5e h1:dVreTP5bOOWt5GFwwvgTE2iU0TkIqi2x3r0b8qGlp6k= +golang.org/x/exp v0.0.0-20190221220918-438050ddec5e/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7 h1:00BeQWmeaGazuOrq8Q5K5d3/cHaGuFrZzpaHBXfrsUA= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3 h1:x/bBzNauLQAlE3fLku/xy92Y8QwKX5HZymrMz2IiKFc= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52 h1:JG/0uqcGdTNgq7FdU+61l5Pdmb8putNZlXb65bJBROs= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b h1:qMK98NmNCRVDIYFycQ5yVRkvgDUFfdP8Ip4KqmDEB7g= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.16.0 h1:dz5IJGuC2BB7qXR5AyHNwAUBhZscK2xVez7mznh72sY= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858 h1:wN+eVZ7U+gqdqkec6C6VXR1OFf9a5Ul9ETzeYsYv20g= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099 h1:XJP7lxbSxWLOMNdBE4B/STaqVy6L73o0knwj2vIlxnw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/googleapis/gax-go/header.go b/vendor/github.com/googleapis/gax-go/header.go new file mode 100644 index 000000000..e1a0af1ba --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/header.go @@ -0,0 +1,40 @@ +// Copyright 2018, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import v2 "github.com/googleapis/gax-go/v2" + +// XGoogHeader is for use by the Google Cloud Libraries only. +// +// XGoogHeader formats key-value pairs. +// The resulting string is suitable for x-goog-api-client header. +func XGoogHeader(keyval ...string) string { + return v2.XGoogHeader(keyval...) +} diff --git a/vendor/github.com/googleapis/gax-go/internal/kokoro/check_incompat_changes.sh b/vendor/github.com/googleapis/gax-go/internal/kokoro/check_incompat_changes.sh new file mode 100755 index 000000000..9236b972b --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/internal/kokoro/check_incompat_changes.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Display commands being run +set -x + +# Only run apidiff checks on go1.12 (we only need it once). +# TODO(deklerk) We should pass an environment variable from kokoro to decide +# this logic instead. +if [[ `go version` != *"go1.12"* ]]; then + exit 0 +fi + +if git log -1 | grep BREAKING_CHANGE_ACCEPTABLE; then + exit 0 +fi + +export GO111MODULE=on + +go install golang.org/x/exp/cmd/apidiff + +# We compare against master@HEAD. This is unfortunate in some cases: if you're +# working on an out-of-date branch, and master gets some new feature (that has +# nothing to do with your work on your branch), you'll get an error message. +# Thankfully the fix is quite simple: rebase your branch. +git clone https://github.com/googleapis/gax-go /tmp/gax + +for dir in "" "/v2"; do + pkg="github.com/googleapis/gax-go$dir" + echo "Testing $pkg" + + # cd to the exact directory that specifies the go module so that it doesn't + # use the module cache. https://go-review.googlesource.com/c/exp/+/155058 + cd "/tmp/gax$dir" + apidiff -w /tmp/pkg.master $pkg + cd - > /dev/null + + # TODO(deklerk) there's probably a nicer way to do this that doesn't require + # two invocations + if ! apidiff -incompatible /tmp/pkg.master $pkg | (! read); then + apidiff -incompatible /tmp/pkg.master $pkg + exit 1 + fi +done diff --git a/vendor/github.com/googleapis/gax-go/internal/kokoro/test.sh b/vendor/github.com/googleapis/gax-go/internal/kokoro/test.sh new file mode 100755 index 000000000..d213839e6 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/internal/kokoro/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# TODO(deklerk) Add integration tests when it's secure to do so. b/64723143 + +# Fail on any error +set -eo pipefail + +# Display commands being run +set -x + +# cd to project dir on Kokoro instance +cd github/gax-go + +go version + +# Set $GOPATH +export GOPATH="$HOME/go" +export GAX_HOME=$GOPATH/src/github.com/googleapis/gax-go +export PATH="$GOPATH/bin:$PATH" +mkdir -p $GAX_HOME + +# Move code into $GOPATH and get dependencies +git clone . $GAX_HOME +cd $GAX_HOME + +try3() { eval "$*" || eval "$*" || eval "$*"; } + +download_deps() { + if [[ `go version` == *"go1.11"* ]] || [[ `go version` == *"go1.12"* ]]; then + export GO111MODULE=on + # All packages, including +build tools, are fetched. + try3 go mod download + else + # Because we don't provide -tags tools, the +build tools + # dependencies aren't fetched. + try3 go get -v -t ./... + fi +} + +download_deps +./internal/kokoro/check_incompat_changes.sh +./internal/kokoro/vet.sh +go test -race -v . 2>&1 | tee $KOKORO_ARTIFACTS_DIR/$KOKORO_GERRIT_CHANGE_NUMBER.txt + +cd v2 +download_deps +go test -race -v . 2>&1 | tee $KOKORO_ARTIFACTS_DIR/$KOKORO_GERRIT_CHANGE_NUMBER.txt diff --git a/vendor/github.com/googleapis/gax-go/internal/kokoro/trampoline.sh b/vendor/github.com/googleapis/gax-go/internal/kokoro/trampoline.sh new file mode 100644 index 000000000..ba17ce014 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/internal/kokoro/trampoline.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eo pipefail +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" diff --git a/vendor/github.com/googleapis/gax-go/internal/kokoro/vet.sh b/vendor/github.com/googleapis/gax-go/internal/kokoro/vet.sh new file mode 100755 index 000000000..f15de65b6 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/internal/kokoro/vet.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Fail on any error +set -eo pipefail + +# Display commands being run +set -x + +# Only run the linter on go1.11, since it needs type aliases (and we only care about its output once). +# TODO(deklerk) We should pass an environment variable from kokoro to decide this logic instead. +if [[ `go version` != *"go1.12"* ]]; then + exit 0 +fi + +export GO111MODULE=on + +go install \ + github.com/golang/protobuf/proto \ + github.com/golang/protobuf/protoc-gen-go \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck + +# Fail if a dependency was added without the necessary go.mod/go.sum change +# being part of the commit. +go mod tidy +git diff go.mod | tee /dev/stderr | (! read) +git diff go.sum | tee /dev/stderr | (! read) + +# Easier to debug CI. +pwd + +# Look at all .go files (ignoring .pb.go files) and make sure they have a Copyright. Fail if any don't. +git ls-files "*[^.pb].go" | xargs grep -L "\(Copyright [0-9]\{4,\}\)" 2>&1 | tee /dev/stderr | (! read) +gofmt -s -d -l . 2>&1 | tee /dev/stderr | (! read) +goimports -l . 2>&1 | tee /dev/stderr | (! read) + +golint ./... 2>&1 | tee /dev/stderr | (! read) +staticcheck ./... diff --git a/vendor/github.com/googleapis/gax-go/invoke.go b/vendor/github.com/googleapis/gax-go/invoke.go new file mode 100644 index 000000000..6422d3f73 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/invoke.go @@ -0,0 +1,52 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + "context" + "time" + + v2 "github.com/googleapis/gax-go/v2" +) + +// APICall is a user defined call stub. +type APICall = v2.APICall + +// Invoke calls the given APICall, +// performing retries as specified by opts, if any. +func Invoke(ctx context.Context, call APICall, opts ...CallOption) error { + return v2.Invoke(ctx, call, opts...) +} + +// Sleep is similar to time.Sleep, but it can be interrupted by ctx.Done() closing. +// If interrupted, Sleep returns ctx.Err(). +func Sleep(ctx context.Context, d time.Duration) error { + return v2.Sleep(ctx, d) +} diff --git a/vendor/github.com/googleapis/gax-go/tools.go b/vendor/github.com/googleapis/gax-go/tools.go new file mode 100644 index 000000000..ffb3ad482 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/tools.go @@ -0,0 +1,33 @@ +// +build tools + +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This package exists to cause `go mod` and `go get` to believe these tools +// are dependencies, even though they are not runtime dependencies of any +// package (these are tools used by our CI builds). This means they will appear +// in our `go.mod` file, but will not be a part of the build. Also, since the +// build target is something non-existent, these should not be included in any +// binaries. + +package gax + +import ( + _ "github.com/golang/protobuf/proto" + _ "github.com/golang/protobuf/protoc-gen-go" + _ "golang.org/x/exp/cmd/apidiff" + _ "golang.org/x/lint/golint" + _ "golang.org/x/tools/cmd/goimports" + _ "honnef.co/go/tools/cmd/staticcheck" +) diff --git a/vendor/github.com/googleapis/gax-go/v2/call_option.go b/vendor/github.com/googleapis/gax-go/v2/call_option.go new file mode 100644 index 000000000..b1d53dd19 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/call_option.go @@ -0,0 +1,161 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + "math/rand" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// CallOption is an option used by Invoke to control behaviors of RPC calls. +// CallOption works by modifying relevant fields of CallSettings. +type CallOption interface { + // Resolve applies the option by modifying cs. + Resolve(cs *CallSettings) +} + +// Retryer is used by Invoke to determine retry behavior. +type Retryer interface { + // Retry reports whether a request should be retriedand how long to pause before retrying + // if the previous attempt returned with err. Invoke never calls Retry with nil error. + Retry(err error) (pause time.Duration, shouldRetry bool) +} + +type retryerOption func() Retryer + +func (o retryerOption) Resolve(s *CallSettings) { + s.Retry = o +} + +// WithRetry sets CallSettings.Retry to fn. +func WithRetry(fn func() Retryer) CallOption { + return retryerOption(fn) +} + +// OnCodes returns a Retryer that retries if and only if +// the previous attempt returns a GRPC error whose error code is stored in cc. +// Pause times between retries are specified by bo. +// +// bo is only used for its parameters; each Retryer has its own copy. +func OnCodes(cc []codes.Code, bo Backoff) Retryer { + return &boRetryer{ + backoff: bo, + codes: append([]codes.Code(nil), cc...), + } +} + +type boRetryer struct { + backoff Backoff + codes []codes.Code +} + +func (r *boRetryer) Retry(err error) (time.Duration, bool) { + st, ok := status.FromError(err) + if !ok { + return 0, false + } + c := st.Code() + for _, rc := range r.codes { + if c == rc { + return r.backoff.Pause(), true + } + } + return 0, false +} + +// Backoff implements exponential backoff. +// The wait time between retries is a random value between 0 and the "retry envelope". +// The envelope starts at Initial and increases by the factor of Multiplier every retry, +// but is capped at Max. +type Backoff struct { + // Initial is the initial value of the retry envelope, defaults to 1 second. + Initial time.Duration + + // Max is the maximum value of the retry envelope, defaults to 30 seconds. + Max time.Duration + + // Multiplier is the factor by which the retry envelope increases. + // It should be greater than 1 and defaults to 2. + Multiplier float64 + + // cur is the current retry envelope + cur time.Duration +} + +// Pause returns the next time.Duration that the caller should use to backoff. +func (bo *Backoff) Pause() time.Duration { + if bo.Initial == 0 { + bo.Initial = time.Second + } + if bo.cur == 0 { + bo.cur = bo.Initial + } + if bo.Max == 0 { + bo.Max = 30 * time.Second + } + if bo.Multiplier < 1 { + bo.Multiplier = 2 + } + // Select a duration between 1ns and the current max. It might seem + // counterintuitive to have so much jitter, but + // https://www.awsarchitectureblog.com/2015/03/backoff.html argues that + // that is the best strategy. + d := time.Duration(1 + rand.Int63n(int64(bo.cur))) + bo.cur = time.Duration(float64(bo.cur) * bo.Multiplier) + if bo.cur > bo.Max { + bo.cur = bo.Max + } + return d +} + +type grpcOpt []grpc.CallOption + +func (o grpcOpt) Resolve(s *CallSettings) { + s.GRPC = o +} + +// WithGRPCOptions allows passing gRPC call options during client creation. +func WithGRPCOptions(opt ...grpc.CallOption) CallOption { + return grpcOpt(append([]grpc.CallOption(nil), opt...)) +} + +// CallSettings allow fine-grained control over how calls are made. +type CallSettings struct { + // Retry returns a Retryer to be used to control retry logic of a method call. + // If Retry is nil or the returned Retryer is nil, the call will not be retried. + Retry func() Retryer + + // CallOptions to be forwarded to GRPC. + GRPC []grpc.CallOption +} diff --git a/vendor/github.com/googleapis/gax-go/v2/call_option_test.go b/vendor/github.com/googleapis/gax-go/v2/call_option_test.go new file mode 100644 index 000000000..6f81305ff --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/call_option_test.go @@ -0,0 +1,88 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var _ Retryer = &boRetryer{} + +func TestBackofDefault(t *testing.T) { + backoff := Backoff{} + + max := []time.Duration{1, 2, 4, 8, 16, 30, 30, 30, 30, 30} + for i, m := range max { + max[i] = m * time.Second + } + + for i, w := range max { + if d := backoff.Pause(); d > w { + t.Errorf("Backoff duration should be at most %s, got %s", w, d) + } else if i < len(max)-1 && backoff.cur != max[i+1] { + t.Errorf("current envelope is %s, want %s", backoff.cur, max[i+1]) + } + } +} + +func TestBackoffExponential(t *testing.T) { + backoff := Backoff{Initial: 1, Max: 20, Multiplier: 2} + want := []time.Duration{1, 2, 4, 8, 16, 20, 20, 20, 20, 20} + for _, w := range want { + if d := backoff.Pause(); d > w { + t.Errorf("Backoff duration should be at most %s, got %s", w, d) + } + } +} + +func TestOnCodes(t *testing.T) { + // Lint errors grpc.Errorf in 1.6. It mistakenly expects the first arg to Errorf to be a string. + errf := status.Errorf + apiErr := errf(codes.Unavailable, "") + tests := []struct { + c []codes.Code + retry bool + }{ + {nil, false}, + {[]codes.Code{codes.DeadlineExceeded}, false}, + {[]codes.Code{codes.DeadlineExceeded, codes.Unavailable}, true}, + {[]codes.Code{codes.Unavailable}, true}, + } + for _, tst := range tests { + b := OnCodes(tst.c, Backoff{}) + if _, retry := b.Retry(apiErr); retry != tst.retry { + t.Errorf("retriable codes: %v, error: %s, retry: %t, want %t", tst.c, apiErr, retry, tst.retry) + } + } +} diff --git a/vendor/github.com/googleapis/gax-go/v2/example_test.go b/vendor/github.com/googleapis/gax-go/v2/example_test.go new file mode 100644 index 000000000..dd3653eff --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/example_test.go @@ -0,0 +1,94 @@ +// Copyright 2019, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax_test + +import ( + "context" + "time" + + "github.com/googleapis/gax-go/v2" + "google.golang.org/grpc/codes" +) + +const someRPCTimeout = 5 * time.Minute + +// Some result that the client might return. +type fakeResponse struct{} + +// Some client that can perform RPCs. +type fakeClient struct{} + +// PerformSomeRPC is a fake RPC that a client might perform. +func (c *fakeClient) PerformSomeRPC(ctx context.Context) (*fakeResponse, error) { + // An actual client would return something meaningful here. + return nil, nil +} + +func ExampleOnCodes() { + ctx := context.Background() + c := &fakeClient{} + + // UNKNOWN and UNAVAILABLE are typically safe to retry for idempotent RPCs. + retryer := gax.OnCodes([]codes.Code{codes.Unknown, codes.Unavailable}, gax.Backoff{ + Initial: time.Second, + Max: 32 * time.Second, + Multiplier: 2, + }) + + performSomeRPCWithRetry := func(ctx context.Context) (*fakeResponse, error) { + for { + resp, err := c.PerformSomeRPC(ctx) + if err != nil { + if delay, shouldRetry := retryer.Retry(err); shouldRetry { + if err := gax.Sleep(ctx, delay); err != nil { + return nil, err + } + continue + } + return nil, err + } + return resp, err + } + } + + // It's recommended to set deadlines on RPCs and around retrying. This is + // also usually preferred over setting some fixed number of retries: one + // advantage this has is that backoff settings can be changed independently + // of the deadline, whereas with a fixed number of retries the deadline + // would be a constantly-shifting goalpost. + ctxWithTimeout, cancel := context.WithDeadline(ctx, time.Now().Add(someRPCTimeout)) + defer cancel() + + resp, err := performSomeRPCWithRetry(ctxWithTimeout) + if err != nil { + // TODO: handle err + } + _ = resp // TODO: use resp if err is nil +} diff --git a/vendor/github.com/googleapis/gax-go/v2/gax.go b/vendor/github.com/googleapis/gax-go/v2/gax.go new file mode 100644 index 000000000..3fd1b0b84 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/gax.go @@ -0,0 +1,39 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Package gax contains a set of modules which aid the development of APIs +// for clients and servers based on gRPC and Google API conventions. +// +// Application code will rarely need to use this library directly. +// However, code generated automatically from API definition files can use it +// to simplify code generation and to provide more convenient and idiomatic API surfaces. +package gax + +// Version specifies the gax-go version being used. +const Version = "2.0.4" diff --git a/vendor/github.com/googleapis/gax-go/v2/go.mod b/vendor/github.com/googleapis/gax-go/v2/go.mod new file mode 100644 index 000000000..9cdfaf447 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/go.mod @@ -0,0 +1,3 @@ +module github.com/googleapis/gax-go/v2 + +require google.golang.org/grpc v1.19.0 diff --git a/vendor/github.com/googleapis/gax-go/v2/go.sum b/vendor/github.com/googleapis/gax-go/v2/go.sum new file mode 100644 index 000000000..7fa23ecf9 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/go.sum @@ -0,0 +1,25 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/googleapis/gax-go/v2/header.go b/vendor/github.com/googleapis/gax-go/v2/header.go new file mode 100644 index 000000000..139371a0b --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/header.go @@ -0,0 +1,53 @@ +// Copyright 2018, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import "bytes" + +// XGoogHeader is for use by the Google Cloud Libraries only. +// +// XGoogHeader formats key-value pairs. +// The resulting string is suitable for x-goog-api-client header. +func XGoogHeader(keyval ...string) string { + if len(keyval) == 0 { + return "" + } + if len(keyval)%2 != 0 { + panic("gax.Header: odd argument count") + } + var buf bytes.Buffer + for i := 0; i < len(keyval); i += 2 { + buf.WriteByte(' ') + buf.WriteString(keyval[i]) + buf.WriteByte('/') + buf.WriteString(keyval[i+1]) + } + return buf.String()[1:] +} diff --git a/vendor/github.com/googleapis/gax-go/v2/header_test.go b/vendor/github.com/googleapis/gax-go/v2/header_test.go new file mode 100644 index 000000000..9d5f6174e --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/header_test.go @@ -0,0 +1,48 @@ +// Copyright 2018, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import "testing" + +func TestXGoogHeader(t *testing.T) { + for _, tst := range []struct { + kv []string + want string + }{ + {nil, ""}, + {[]string{"abc", "def"}, "abc/def"}, + {[]string{"abc", "def", "xyz", "123", "foo", ""}, "abc/def xyz/123 foo/"}, + } { + got := XGoogHeader(tst.kv...) + if got != tst.want { + t.Errorf("Header(%q) = %q, want %q", tst.kv, got, tst.want) + } + } +} diff --git a/vendor/github.com/googleapis/gax-go/v2/invoke.go b/vendor/github.com/googleapis/gax-go/v2/invoke.go new file mode 100644 index 000000000..fe31dd004 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/invoke.go @@ -0,0 +1,99 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + "context" + "strings" + "time" +) + +// APICall is a user defined call stub. +type APICall func(context.Context, CallSettings) error + +// Invoke calls the given APICall, +// performing retries as specified by opts, if any. +func Invoke(ctx context.Context, call APICall, opts ...CallOption) error { + var settings CallSettings + for _, opt := range opts { + opt.Resolve(&settings) + } + return invoke(ctx, call, settings, Sleep) +} + +// Sleep is similar to time.Sleep, but it can be interrupted by ctx.Done() closing. +// If interrupted, Sleep returns ctx.Err(). +func Sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + select { + case <-ctx.Done(): + t.Stop() + return ctx.Err() + case <-t.C: + return nil + } +} + +type sleeper func(ctx context.Context, d time.Duration) error + +// invoke implements Invoke, taking an additional sleeper argument for testing. +func invoke(ctx context.Context, call APICall, settings CallSettings, sp sleeper) error { + var retryer Retryer + for { + err := call(ctx, settings) + if err == nil { + return nil + } + if settings.Retry == nil { + return err + } + // Never retry permanent certificate errors. (e.x. if ca-certificates + // are not installed). We should only make very few, targeted + // exceptions: many (other) status=Unavailable should be retried, such + // as if there's a network hiccup, or the internet goes out for a + // minute. This is also why here we are doing string parsing instead of + // simply making Unavailable a non-retried code elsewhere. + if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") { + return err + } + if retryer == nil { + if r := settings.Retry(); r != nil { + retryer = r + } else { + return err + } + } + if d, ok := retryer.Retry(err); !ok { + return err + } else if err = sp(ctx, d); err != nil { + return err + } + } +} diff --git a/vendor/github.com/googleapis/gax-go/v2/invoke_test.go b/vendor/github.com/googleapis/gax-go/v2/invoke_test.go new file mode 100644 index 000000000..d476d14e9 --- /dev/null +++ b/vendor/github.com/googleapis/gax-go/v2/invoke_test.go @@ -0,0 +1,155 @@ +// Copyright 2016, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package gax + +import ( + "context" + "errors" + "testing" + "time" +) + +var canceledContext context.Context + +func init() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + canceledContext = ctx +} + +// recordSleeper is a test implementation of sleeper. +type recordSleeper int + +func (s *recordSleeper) sleep(ctx context.Context, _ time.Duration) error { + *s++ + return ctx.Err() +} + +type boolRetryer bool + +func (r boolRetryer) Retry(err error) (time.Duration, bool) { return 0, bool(r) } + +func TestInvokeSuccess(t *testing.T) { + apiCall := func(context.Context, CallSettings) error { return nil } + var sp recordSleeper + err := invoke(context.Background(), apiCall, CallSettings{}, sp.sleep) + + if err != nil { + t.Errorf("found error %s, want nil", err) + } + if sp != 0 { + t.Errorf("slept %d times, should not have slept since the call succeeded", int(sp)) + } +} + +func TestInvokeNoRetry(t *testing.T) { + apiErr := errors.New("foo error") + apiCall := func(context.Context, CallSettings) error { return apiErr } + var sp recordSleeper + err := invoke(context.Background(), apiCall, CallSettings{}, sp.sleep) + + if err != apiErr { + t.Errorf("found error %s, want %s", err, apiErr) + } + if sp != 0 { + t.Errorf("slept %d times, should not have slept since retry is not specified", int(sp)) + } +} + +func TestInvokeNilRetry(t *testing.T) { + apiErr := errors.New("foo error") + apiCall := func(context.Context, CallSettings) error { return apiErr } + var settings CallSettings + WithRetry(func() Retryer { return nil }).Resolve(&settings) + var sp recordSleeper + err := invoke(context.Background(), apiCall, settings, sp.sleep) + + if err != apiErr { + t.Errorf("found error %s, want %s", err, apiErr) + } + if sp != 0 { + t.Errorf("slept %d times, should not have slept since retry is not specified", int(sp)) + } +} + +func TestInvokeNeverRetry(t *testing.T) { + apiErr := errors.New("foo error") + apiCall := func(context.Context, CallSettings) error { return apiErr } + var settings CallSettings + WithRetry(func() Retryer { return boolRetryer(false) }).Resolve(&settings) + var sp recordSleeper + err := invoke(context.Background(), apiCall, settings, sp.sleep) + + if err != apiErr { + t.Errorf("found error %s, want %s", err, apiErr) + } + if sp != 0 { + t.Errorf("slept %d times, should not have slept since retry is not specified", int(sp)) + } +} + +func TestInvokeRetry(t *testing.T) { + const target = 3 + + retryNum := 0 + apiErr := errors.New("foo error") + apiCall := func(context.Context, CallSettings) error { + retryNum++ + if retryNum < target { + return apiErr + } + return nil + } + var settings CallSettings + WithRetry(func() Retryer { return boolRetryer(true) }).Resolve(&settings) + var sp recordSleeper + err := invoke(context.Background(), apiCall, settings, sp.sleep) + + if err != nil { + t.Errorf("found error %s, want nil, call should have succeeded after %d tries", err, target) + } + if sp != target-1 { + t.Errorf("retried %d times, want %d", int(sp), int(target-1)) + } +} + +func TestInvokeRetryTimeout(t *testing.T) { + apiErr := errors.New("foo error") + apiCall := func(context.Context, CallSettings) error { return apiErr } + var settings CallSettings + WithRetry(func() Retryer { return boolRetryer(true) }).Resolve(&settings) + var sp recordSleeper + + err := invoke(canceledContext, apiCall, settings, sp.sleep) + + if err != context.Canceled { + t.Errorf("found error %s, want %s", err, context.Canceled) + } +}