buildah: support for --retry and --retry-delay for push/pull failures

Allows users to configure `--retry` attempts and `--retry-delay`
duration using two additional flags for commands

* buildah build
* buildah pull
* buildah push
* buildah from
* buildah add
* buildah copy

Closes: https://github.com/containers/buildah/issues/4018

Not sure how we can test retry attempts in CI, but added a test in
tests/bud.bats which verfies we parse and added flag in other options
for sanity parsing checking.

Signed-off-by: Aditya R <arajan@redhat.com>

cmd/buildah/addcopy.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/buildah/internal/util"
@@ -34,6 +35,8 @@ type addCopyResults struct {
 	creds            string
 	tlsVerify        bool
 	certDir          string
+	retry            int
+	retryDelay       string
 }
 
 func createCommand(addCopy string, desc string, short string, opts *addCopyResults) *cobra.Command {
@@ -78,6 +81,8 @@ func applyFlagVars(flags *pflag.FlagSet, opts *addCopyResults) {
 	}
 	flags.StringVar(&opts.ignoreFile, "ignorefile", "", "path to .containerignore file")
 	flags.StringVar(&opts.contextdir, "contextdir", "", "context directory path")
+	flags.IntVar(&opts.retry, "retry", buildahcli.MaxPullPushRetries, "number of times to retry in case of failure when performing pull")
+	flags.StringVar(&opts.retryDelay, "retry-delay", buildahcli.PullPushRetryDelay.String(), "delay between retries in case of pull failures")
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "don't output a digest of the newly-added/copied content")
 	flags.BoolVar(&opts.tlsVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing registries when pulling images. TLS verification cannot be used when talking to an insecure registry.")
 	if err := flags.MarkHidden("tls-verify"); err != nil {
@@ -165,13 +170,18 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe
 			if err2 != nil {
 				return fmt.Errorf("unable to obtain decrypt config: %w", err2)
 			}
+			var pullPushRetryDelay time.Duration
+			pullPushRetryDelay, err = time.ParseDuration(iopts.retryDelay)
+			if err != nil {
+				return fmt.Errorf("unable to parse value provided %q as --retry-delay: %w", iopts.retryDelay, err)
+			}
 			options := buildah.BuilderOptions{
 				FromImage:           iopts.from,
 				BlobDirectory:       iopts.blobCache,
 				SignaturePolicyPath: iopts.signaturePolicy,
 				SystemContext:       systemContext,
-				MaxPullRetries:      buildahcli.MaxPullPushRetries,
-				PullRetryDelay:      buildahcli.PullPushRetryDelay,
+				MaxPullRetries:      iopts.retry,
+				PullRetryDelay:      pullPushRetryDelay,
 				OciDecryptConfig:    decryptConfig,
 			}
 			if !iopts.quiet {

cmd/buildah/from.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/buildah/define"
@@ -301,6 +302,12 @@ func fromCmd(c *cobra.Command, args []string, iopts fromReply) error {
 		return fmt.Errorf("unable to obtain decrypt config: %w", err)
 	}
 
+	var pullPushRetryDelay time.Duration
+	pullPushRetryDelay, err = time.ParseDuration(iopts.RetryDelay)
+	if err != nil {
+		return fmt.Errorf("unable to parse value provided %q as --retry-delay: %w", iopts.RetryDelay, err)
+	}
+
 	options := buildah.BuilderOptions{
 		FromImage:             args[0],
 		Container:             iopts.name,
@@ -320,8 +327,8 @@ func fromCmd(c *cobra.Command, args []string, iopts fromReply) error {
 		Format:                format,
 		BlobDirectory:         iopts.BlobCache,
 		Devices:               devices,
-		MaxPullRetries:        buildahcli.MaxPullPushRetries,
-		PullRetryDelay:        buildahcli.PullPushRetryDelay,
+		MaxPullRetries:        iopts.Retry,
+		PullRetryDelay:        pullPushRetryDelay,
 		OciDecryptConfig:      decConfig,
 	}
 

cmd/buildah/pull.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/containers/buildah"
 	"github.com/containers/buildah/define"
@@ -28,6 +29,8 @@ type pullOptions struct {
 	tlsVerify        bool
 	decryptionKeys   []string
 	pullPolicy       string
+	retry            int
+	retryDelay       string
 }
 
 func init() {
@@ -72,6 +75,8 @@ func init() {
 	flags.StringSlice("platform", []string{parse.DefaultPlatform()}, "prefer OS/ARCH instead of the current operating system and architecture for choosing images")
 	flags.String("variant", "", "override the `variant` of the specified image")
 	flags.BoolVar(&opts.tlsVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry. TLS verification cannot be used when talking to an insecure registry.")
+	flags.IntVar(&opts.retry, "retry", buildahcli.MaxPullPushRetries, "number of times to retry in case of failure when performing pull")
+	flags.StringVar(&opts.retryDelay, "retry-delay", buildahcli.PullPushRetryDelay.String(), "delay between retries in case of pull failures")
 	if err := flags.MarkHidden("blob-cache"); err != nil {
 		panic(fmt.Sprintf("error marking blob-cache as hidden: %v", err))
 	}
@@ -119,6 +124,11 @@ func pullCmd(c *cobra.Command, args []string, iopts pullOptions) error {
 	if !ok {
 		return fmt.Errorf("unsupported pull policy %q", iopts.pullPolicy)
 	}
+	var pullPushRetryDelay time.Duration
+	pullPushRetryDelay, err = time.ParseDuration(iopts.retryDelay)
+	if err != nil {
+		return fmt.Errorf("unable to parse value provided %q as --retry-delay: %w", iopts.retryDelay, err)
+	}
 	options := buildah.PullOptions{
 		SignaturePolicyPath: iopts.signaturePolicy,
 		Store:               store,
@@ -127,8 +137,8 @@ func pullCmd(c *cobra.Command, args []string, iopts pullOptions) error {
 		AllTags:             iopts.allTags,
 		ReportWriter:        os.Stderr,
 		RemoveSignatures:    iopts.removeSignatures,
-		MaxRetries:          buildahcli.MaxPullPushRetries,
-		RetryDelay:          buildahcli.PullPushRetryDelay,
+		MaxRetries:          iopts.retry,
+		RetryDelay:          pullPushRetryDelay,
 		OciDecryptConfig:    decConfig,
 		PullPolicy:          policy,
 	}

cmd/buildah/push.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"time"
 
 	"errors"
 
@@ -36,6 +37,8 @@ type pushOptions struct {
 	format             string
 	compressionFormat  string
 	compressionLevel   int
+	retry              int
+	retryDelay         string
 	rm                 bool
 	quiet              bool
 	removeSignatures   bool
@@ -88,6 +91,8 @@ func init() {
 	flags.StringVar(&opts.compressionFormat, "compression-format", "", "compression format to use")
 	flags.IntVar(&opts.compressionLevel, "compression-level", 0, "compression level to use")
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "don't output progress information when pushing images")
+	flags.IntVar(&opts.retry, "retry", buildahcli.MaxPullPushRetries, "number of times to retry in case of failure when performing push/pull")
+	flags.StringVar(&opts.retryDelay, "retry-delay", buildahcli.PullPushRetryDelay.String(), "delay between retries in case of push/pull failures")
 	flags.BoolVar(&opts.rm, "rm", false, "remove the manifest list if push succeeds")
 	flags.BoolVarP(&opts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing image")
 	flags.StringVar(&opts.signBy, "sign-by", "", "sign the image using a GPG key with the specified `FINGERPRINT`")
@@ -188,6 +193,12 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
 		return fmt.Errorf("unable to obtain encryption config: %w", err)
 	}
 
+	var pullPushRetryDelay time.Duration
+	pullPushRetryDelay, err = time.ParseDuration(iopts.retryDelay)
+	if err != nil {
+		return fmt.Errorf("unable to parse value provided %q as --retry-delay: %w", iopts.retryDelay, err)
+	}
+
 	options := buildah.PushOptions{
 		Compression:         compress,
 		ManifestType:        manifestType,
@@ -197,8 +208,8 @@ func pushCmd(c *cobra.Command, args []string, iopts pushOptions) error {
 		BlobDirectory:       iopts.blobCache,
 		RemoveSignatures:    iopts.removeSignatures,
 		SignBy:              iopts.signBy,
-		MaxRetries:          buildahcli.MaxPullPushRetries,
-		RetryDelay:          buildahcli.PullPushRetryDelay,
+		MaxRetries:          iopts.retry,
+		RetryDelay:          pullPushRetryDelay,
 		OciEncryptConfig:    encConfig,
 		OciEncryptLayers:    encLayers,
 	}

docs/buildah-add.1.md

@@ -52,6 +52,18 @@ Path to an alternative .containerignore (.dockerignore) file. Requires \-\-conte
 
 Refrain from printing a digest of the added content.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing pull of images from registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing pull of images from registry.
+
+Defaults to `2s`.
+
 ## EXAMPLE
 
 buildah add containerID '/myapp/app.conf' '/myapp/app.conf'

docs/buildah-build.1.md

@@ -612,6 +612,18 @@ Suppress output messages which indicate which instruction is being processed,
 and of progress when pulling images from a registry, and when writing the
 output image.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing push/pull of images to/from registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing push/pull of images to/from registry.
+
+Defaults to `2s`.
+
 **--rm** *bool-value*
 
 Remove intermediate containers after a successful build (default true).

docs/buildah-copy.1.md

@@ -50,6 +50,18 @@ Path to an alternative .containerignore (.dockerignore) file. Requires \-\-conte
 
 Refrain from printing a digest of the copied content.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing pull of images from registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing pull of images from registry.
+
+Defaults to `2s`.
+
 ## EXAMPLE
 
 buildah copy containerID '/myapp/app.conf' '/myapp/app.conf'

docs/buildah-from.1.md

@@ -333,6 +333,18 @@ Defaults to *true*.
 
 If an image needs to be pulled from the registry, suppress progress output.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing pull of images from registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing pull of images from registry.
+
+Defaults to `2s`.
+
 **--security-opt**=[]
 
 Security Options

docs/buildah-pull.1.md

@@ -87,6 +87,18 @@ If an image needs to be pulled from the registry, suppress progress output.
 
 Don't copy signatures when pulling images.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing pull of images from registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing pull of images from registry.
+
+Defaults to `2s`.
+
 **--tls-verify** *bool-value*
 
 Require HTTPS and verification of certificates when talking to container registries (defaults to true).  TLS verification cannot be used when talking to an insecure registry.

docs/buildah-push.1.md

@@ -82,6 +82,18 @@ When writing the output image, suppress progress output.
 
 Don't copy signatures when pushing images.
 
+**--retry** *attempts*
+
+Number of times to retry in case of failure when performing push of images to registry.
+
+Defaults to `3`.
+
+**--retry-delay** *duration*
+
+Duration of delay between retry attempts in case of failure when performing push of images to registry.
+
+Defaults to `2s`.
+
 **--rm**
 
 When pushing a manifest list or image index, delete them from local storage if pushing succeeds.

pkg/cli/build.go

@@ -310,6 +310,14 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) (
 			return options, nil, nil, fmt.Errorf("unable to parse value provided %q as --cache-ttl: %w", iopts.CacheTTL, err)
 		}
 	}
+	var pullPushRetryDelay time.Duration
+	pullPushRetryDelay, err = time.ParseDuration(iopts.RetryDelay)
+	if err != nil {
+		return options, nil, nil, fmt.Errorf("unable to parse value provided %q as --retry-delay: %w", iopts.RetryDelay, err)
+	}
+	// Following log line is used in integration test.
+	logrus.Debugf("Setting MaxPullPushRetries to %d and PullPushRetryDelay to %v", iopts.Retry, pullPushRetryDelay)
+
 	options = define.BuildOptions{
 		AddCapabilities:         iopts.CapAdd,
 		AdditionalBuildContexts: additionalBuildContext,
@@ -349,7 +357,7 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) (
 		LogRusage:               iopts.LogRusage,
 		LogSplitByPlatform:      iopts.LogSplitByPlatform,
 		Manifest:                iopts.Manifest,
-		MaxPullPushRetries:      MaxPullPushRetries,
+		MaxPullPushRetries:      iopts.Retry,
 		NamespaceOptions:        namespaceOptions,
 		NoCache:                 iopts.NoCache,
 		OS:                      systemContext.OSChoice,
@@ -361,7 +369,7 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) (
 		OutputFormat:            format,
 		Platforms:               platforms,
 		PullPolicy:              pullPolicy,
-		PullPushRetryDelay:      PullPushRetryDelay,
+		PullPushRetryDelay:      pullPushRetryDelay,
 		Quiet:                   iopts.Quiet,
 		RemoveIntermediateCtrs:  iopts.Rm,
 		ReportWriter:            reporter,

pkg/cli/common.go

@@ -125,6 +125,8 @@ type FromAndBudResults struct {
 	Isolation      string
 	Memory         string
 	MemorySwap     string
+	Retry          int
+	RetryDelay     string
 	SecurityOpt    []string
 	ShmSize        string
 	Ulimit         []string
@@ -344,6 +346,8 @@ func GetFromAndBudFlags(flags *FromAndBudResults, usernsResults *UserNSResults,
 	fs.StringVar(&flags.Isolation, "isolation", DefaultIsolation(), "`type` of process isolation to use. Use BUILDAH_ISOLATION environment variable to override.")
 	fs.StringVarP(&flags.Memory, "memory", "m", "", "memory limit (format: <number>[<unit>], where unit = b, k, m or g)")
 	fs.StringVar(&flags.MemorySwap, "memory-swap", "", "swap limit equal to memory plus swap: '-1' to enable unlimited swap")
+	fs.IntVar(&flags.Retry, "retry", MaxPullPushRetries, "number of times to retry in case of failure when performing push/pull")
+	fs.StringVar(&flags.RetryDelay, "retry-delay", PullPushRetryDelay.String(), "delay between retries in case of push/pull failures")
 	fs.String("arch", runtime.GOARCH, "set the ARCH of the image to the provided value instead of the architecture of the host")
 	fs.String("os", runtime.GOOS, "prefer `OS` instead of the running OS when pulling images")
 	fs.StringSlice("platform", []string{parse.DefaultPlatform()}, "set the OS/ARCH/VARIANT of the image to the provided value instead of the current operating system and architecture of the host (for example `linux/arm`)")
@@ -386,6 +390,8 @@ func GetFromAndBudFlagsCompletions() commonComp.FlagCompletions {
 	flagCompletion["memory-swap"] = commonComp.AutocompleteNone
 	flagCompletion["os"] = commonComp.AutocompleteNone
 	flagCompletion["platform"] = commonComp.AutocompleteNone
+	flagCompletion["retry"] = commonComp.AutocompleteNone
+	flagCompletion["retry-delay"] = commonComp.AutocompleteNone
 	flagCompletion["security-opt"] = commonComp.AutocompleteNone
 	flagCompletion["shm-size"] = commonComp.AutocompleteNone
 	flagCompletion["ulimit"] = commonComp.AutocompleteNone

tests/add.bats

@@ -24,7 +24,7 @@ load helpers
   mkdir $root/subdir $root/other-subdir
   # Copy a file to the working directory
   run_buildah config --workingdir=/ $cid
-  run_buildah add $cid ${TEST_SCRATCH_DIR}/randomfile
+  run_buildah add --retry 4 --retry-delay 4s $cid ${TEST_SCRATCH_DIR}/randomfile
   # Copy a file to a specific subdirectory
   run_buildah add $cid ${TEST_SCRATCH_DIR}/randomfile /subdir
   # Copy two files to a specific subdirectory

tests/bud.bats

@@ -463,6 +463,19 @@ _EOF
   assert "$output" !~ "unwanted stage"
 }
 
+@test "build test --retry and --retry-delay" {
+  mkdir -p ${TEST_SCRATCH_DIR}/bud/platform
+
+  echo something > ${TEST_SCRATCH_DIR}/bud/platform/somefile
+  cat > ${TEST_SCRATCH_DIR}/bud/platform/Dockerfile << _EOF
+FROM alpine
+RUN echo hello
+_EOF
+
+  run_buildah --log-level debug build --retry 4 --retry-delay 5s $WITH_POLICY_JSON --layers -t source -f ${TEST_SCRATCH_DIR}/bud/platform/Dockerfile ${TEST_SCRATCH_DIR}/bud/platform
+  expect_output --substring "Setting MaxPullPushRetries to 4 and PullPushRetryDelay to 5s"
+}
+
 # Test skipping unwanted stage with COPY from stage index
 @test "build-test skipping unwanted stages with COPY from stage index" {
   mkdir -p ${TEST_SCRATCH_DIR}/bud/platform

tests/copy.bats

@@ -24,7 +24,7 @@ load helpers
   root=$output
   run_buildah config --workingdir / $cid
   # copy ${TEST_SCRATCH_DIR}/randomfile to a file of the same name in the container's working directory
-  run_buildah copy $cid ${TEST_SCRATCH_DIR}/randomfile
+  run_buildah copy --retry 4 --retry-delay 4s $cid ${TEST_SCRATCH_DIR}/randomfile
   # copy ${TEST_SCRATCH_DIR}/other-randomfile and ${TEST_SCRATCH_DIR}/third-randomfile to a new directory named ${TEST_SCRATCH_DIR}/randomfile in the container
   run_buildah copy $cid ${TEST_SCRATCH_DIR}/other-randomfile ${TEST_SCRATCH_DIR}/third-randomfile ${TEST_SCRATCH_DIR}/randomfile
   # try to copy ${TEST_SCRATCH_DIR}/other-randomfile and ${TEST_SCRATCH_DIR}/third-randomfile to a /randomfile, which already exists and is a file

tests/from.bats

@@ -35,7 +35,7 @@ load helpers
   elsewhere=${TEST_SCRATCH_DIR}/elsewhere-img
   mkdir -p ${elsewhere}
 
-  run_buildah from --pull $WITH_POLICY_JSON scratch
+  run_buildah from --retry 4 --retry-delay 4s --pull $WITH_POLICY_JSON scratch
   cid=$output
   run_buildah commit $WITH_POLICY_JSON $cid dir:${elsewhere}
   run_buildah rm $cid

tests/pull.bats

@@ -18,7 +18,7 @@ load helpers
 }
 
 @test "pull-flags-order-verification" {
-  run_buildah 125 pull image1 --tls-verify
+  run_buildah 125 pull --retry 4 --retry-delay 4s image1 --tls-verify
   check_options_flag_err "--tls-verify"
 
   run_buildah 125 pull image1 --authfile=/tmp/somefile

tests/push.bats

@@ -45,7 +45,7 @@ load helpers
   _prefetch alpine
   run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine
   cid=$output
-  run_buildah push $WITH_POLICY_JSON --format oci alpine dir:$mytmpdir
+  run_buildah push --retry 4 --retry-delay 4s $WITH_POLICY_JSON --format oci alpine dir:$mytmpdir
   run cat $mytmpdir/manifest.json
   expect_output --substring "application/vnd.oci.image.config.v1\\+json"