From f79d56bd13c810c82bcd961c72bb98a8ea092808 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Mon, 23 Jan 2017 15:25:38 -0600 Subject: [PATCH] Disconnect backends Ideally, the atomic CLI should be able to operate independently of the backends it supports. For example, if dockerd is inactive, the ostree backend and atomic cli should still work. This requires some tweaking to the backendutils code and the work flow. We also need to specifically know if the user passes --storage so that we treat that as an explicit override. The work flow is now roughly: * a default storage can be defined in atomic.conf (was always this way) * if not defined, defaults to docker. * if --storage is passed, treat explictly and fail if cannot execute * if no --storage is specified, use default. if default is not available, move onto the next backend. --- Atomic/backends/_docker.py | 9 +++++- Atomic/backends/_ostree.py | 3 ++ Atomic/backends/backend.py | 5 +++ Atomic/backendutils.py | 66 ++++++++++++++++++++++++++++++-------- Atomic/containers.py | 19 ++++++++--- Atomic/delete.py | 5 ++- Atomic/discovery.py | 8 +++-- Atomic/images.py | 3 +- Atomic/info.py | 11 ++++--- Atomic/objects/image.py | 2 +- Atomic/run.py | 2 +- Atomic/syscontainers.py | 4 +++ Atomic/update.py | 4 +-- Atomic/util.py | 5 ++- Atomic/verify.py | 4 +-- bash/atomic | 1 + 16 files changed, 115 insertions(+), 36 deletions(-) diff --git a/Atomic/backends/_docker.py b/Atomic/backends/_docker.py index d9ab1d2..bc8d448 100644 --- a/Atomic/backends/_docker.py +++ b/Atomic/backends/_docker.py @@ -22,7 +22,14 @@ class DockerBackend(Backend): def __init__(self): self.input = None self._d = None - self._ping() + + @property + def available(self): + try: + _ = self.d + return True + except util.NoDockerDaemon: + return False @property def d(self): diff --git a/Atomic/backends/_ostree.py b/Atomic/backends/_ostree.py index e2971c3..c1fb7ba 100644 --- a/Atomic/backends/_ostree.py +++ b/Atomic/backends/_ostree.py @@ -24,6 +24,9 @@ class OSTreeBackend(Backend): def backend(self): return "ostree" + def available(self): + return self.syscontainers.available + def _make_container(self, info): container_id = info['Id'] runtime = self.syscontainers.get_container_runtime_info(container_id) diff --git a/Atomic/backends/backend.py b/Atomic/backends/backend.py index 005a63c..d44c752 100644 --- a/Atomic/backends/backend.py +++ b/Atomic/backends/backend.py @@ -144,3 +144,8 @@ class Backend(object): #pylint: disable=metaclass-assignment pass + @abstractproperty + def available(self): + pass + + diff --git a/Atomic/backendutils.py b/Atomic/backendutils.py index cea2b2e..a98a9db 100644 --- a/Atomic/backendutils.py +++ b/Atomic/backendutils.py @@ -1,6 +1,9 @@ from Atomic.backends._docker import DockerBackend from Atomic.backends._ostree import OSTreeBackend +from Atomic.util import write_out, get_atomic_config +ATOMIC_CONFIG = get_atomic_config() +default_storage = ATOMIC_CONFIG.get('default_storage', "docker") class BackendUtils(object): """ @@ -9,13 +12,40 @@ class BackendUtils(object): BACKENDS = [DockerBackend, OSTreeBackend] - def get_backend_from_string(self, str_backend): + @property + def available_backends(self): + return self._set_available_backends() + + def _set_available_backends(self): + bes = [] + for x in self.BACKENDS: + be = x() + if be.available: + bes.append(x) + if len(bes) < 1: + raise ValueError("No backends are enabled for Atomic.") + return bes + + def dump_backends(self): + backends = '' + for i in self.available_backends: + be = i() + backends += "{}: Active, ".format(be.backend) + write_out("Backends({})\n".format(backends)) + + def get_backend_from_string(self, str_backend, init=True): for _backend in self.BACKENDS: + backend = _backend backend_obj = _backend() if backend_obj.backend == str_backend: - return backend_obj + if init: + return backend_obj + return backend raise ValueError("Unable to associate string '{}' with backend".format(str_backend)) + def _get_backend(self, backend): + return self.get_backend_from_string(backend, init=False) + def _get_backend_index_from_string(self, str_backend): return [x().backend for x in self.BACKENDS].index(str_backend) @@ -27,7 +57,7 @@ class BackendUtils(object): def backend_has_container(backend, container): return True if backend.has_container(container) else False - def get_backend_and_image_obj(self, img, str_preferred_backend=None): + def get_backend_and_image_obj(self, img, str_preferred_backend=None, required=False): """ Given an image name (str) and optionally a str reference to a backend, this method looks for the image firstly on the preferred backend and @@ -37,14 +67,18 @@ class BackendUtils(object): :param str_preferred_backend: i.e. 'docker' :return: backend object and image object """ - backends = list(self.BACKENDS) + backends = list(self.available_backends) + + if str_preferred_backend and self._get_backend(str_preferred_backend) not in self.available_backends and required: + raise ValueError("The '{}' backend appears unavailable/inactive".format(str_preferred_backend)) # Check preferred backend first - if str_preferred_backend: + if str_preferred_backend and self._get_backend(str_preferred_backend) in self.available_backends: be = self.get_backend_from_string(str_preferred_backend) img_obj = be.has_image(img) if img_obj: return be, img_obj - + if required: + raise ValueError("Unable to find {} in the {} backend".format(img, str_preferred_backend)) # Didnt find in preferred, need to remove it from the list now del backends[self._get_backend_index_from_string(str_preferred_backend)] @@ -59,11 +93,11 @@ class BackendUtils(object): if len(img_in_backends) == 1: return img_in_backends[0] if len(img_in_backends) == 0: - raise ValueError("Unable to find backend associated with image '{}'".format(img)) + raise ValueError("Unable to find '{}' in the following backends: {}".format(img, ", ".join([x().backend for x in self.available_backends]))) raise ValueError("Found {} in multiple storage backends: {}". - format(img, ', '.join([x.backend for x in img_in_backends]))) + format(img, ', '.join([x.backend for x, _ in img_in_backends]))) - def get_backend_and_container_obj(self, container_name, str_preferred_backend=None): + def get_backend_and_container_obj(self, container_name, str_preferred_backend=None, required=False): """ Given a container name (str) and optionally a str reference to a backend, this method looks for the container firstly on the preferred backend and @@ -73,13 +107,19 @@ class BackendUtils(object): :param str_preferred_backend: i.e. 'docker' :return: backend object and container object """ - backends = list(self.BACKENDS) + + if str_preferred_backend and self._get_backend(str_preferred_backend) not in self.available_backends and required: + raise ValueError("The '{}' backend appears unavailable/inactive".format(str_preferred_backend)) + + backends = list(self.available_backends) # Check preferred backend first - if str_preferred_backend: + if str_preferred_backend and self._get_backend(str_preferred_backend) in self.available_backends: be = self.get_backend_from_string(str_preferred_backend) con_obj = be.has_container(container_name) if con_obj: return be, con_obj + if required: + raise ValueError("Unable to find {} in the {} backend".format(container_name, str_preferred_backend)) # Didnt find in preferred, need to remove it from the list now del backends[self._get_backend_index_from_string(str_preferred_backend)] @@ -97,7 +137,7 @@ class BackendUtils(object): format(container_name, ', '.join([x.backend for x in container_in_backends]))) def get_images(self, get_all=False): - backends = self.BACKENDS + backends = self.available_backends img_objs = [] for backend in backends: be = backend() @@ -105,7 +145,7 @@ class BackendUtils(object): return img_objs def get_containers(self): - backends = self.BACKENDS + backends = self.available_backends con_objs = [] for backend in backends: be = backend() diff --git a/Atomic/containers.py b/Atomic/containers.py index bdbd8ab..fe522a1 100644 --- a/Atomic/containers.py +++ b/Atomic/containers.py @@ -33,6 +33,10 @@ def cli(subparser): delete_parser.add_argument("-a", "--all", action='store_true',dest="all", default=False, help=_("Delete all containers")) + delete_parser.add_argument("--storage", default=None, dest="storage", + help=_("Specify the storage from which to delete the container from. " + "If not specified and there are containers with the same name in " + "different storages, you will be prompted to specify.")) delete_parser.add_argument("containers", nargs='*', help=_("Specify one or more containers. Must be final arguments.")) delete_parser.set_defaults(_class=Containers, func='delete') @@ -86,6 +90,10 @@ class Containers(Atomic): FILTER_KEYWORDS= {"container": "id", "image": "image_name", "command": "command", "created": "created", "state": "state", "runtime": "runtime", "backend" : "backend.backend"} + def __init__(self): + super(Containers, self).__init__() + self.beu = BackendUtils() + def fstrim(self): with AtomicDocker() as client: for container in client.containers(): @@ -120,6 +128,8 @@ class Containers(Atomic): def ps_tty(self): if self.args.debug: util.write_out(str(self.args)) + self.beu.dump_backends() + container_objects = self._ps() if not any([x.running for x in container_objects]) and not self.args.all: @@ -179,8 +189,7 @@ class Containers(Atomic): raise ValueError("The filter {} is not valid. " "Please choose from {}".format(_filter, [x for x in self.FILTER_KEYWORDS])) _check_filters() - beu = BackendUtils() - containers = self.filter_container_objects(beu.get_containers()) + containers = self.filter_container_objects(self.beu.get_containers()) self._mark_vulnerable(containers) if self.args.all: return containers @@ -207,17 +216,17 @@ class Containers(Atomic): def delete(self): if self.args.debug: util.write_out(str(self.args)) + self.beu.dump_backends() if len(self.args.containers) > 0 and self.args.all: raise ValueError("You must select --all or provide a list of containers to delete.") - beu = BackendUtils() if self.args.all: - container_objects = beu.get_containers() + container_objects = self.beu.get_containers() else: container_objects = [] for con in self.args.containers: - _, con_obj = beu.get_backend_and_container_obj(con, str_preferred_backend=storage) + _, con_obj = self.beu.get_backend_and_container_obj(con, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) container_objects.append(con_obj) four_col = " {0:12} {1:20} {2:25} {3:10}" diff --git a/Atomic/delete.py b/Atomic/delete.py index 4ad468b..b4c8cfe 100644 --- a/Atomic/delete.py +++ b/Atomic/delete.py @@ -3,6 +3,9 @@ from . import util import sys from Atomic.backendutils import BackendUtils +ATOMIC_CONFIG = util.get_atomic_config() +storage = ATOMIC_CONFIG.get('default_storage', "docker") + class Delete(Atomic): def __init__(self): super(Delete, self).__init__() @@ -31,7 +34,7 @@ class Delete(Atomic): delete_objects = beu.get_images(get_all=True) else: for image in self.args.delete_targets: - _, img_obj = beu.get_backend_and_image_obj(image, str_preferred_backend=self.args.storage) + _, img_obj = beu.get_backend_and_image_obj(image, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) delete_objects.append(img_obj) if self.args.remote: diff --git a/Atomic/discovery.py b/Atomic/discovery.py index bf1a2c3..c4d541f 100644 --- a/Atomic/discovery.py +++ b/Atomic/discovery.py @@ -55,7 +55,7 @@ class RegistryInspect(): fqdn += ":{}".format(self.tag) return fqdn - def find_image_on_registry(self): + def find_image_on_registry(self, quiet=False): """ Find the fully qualified image name for given input when registry is unknown @@ -68,13 +68,15 @@ class RegistryInspect(): registries = [i['name'] for i in [x for x in self.registries if x['search']]] for registry in registries: fqdn = self.assemble_fqdn(registry=registry, include_tag=True) - util.write_out("Trying {}...".format(fqdn)) + if not quiet: + util.write_out("Trying {}...".format(fqdn)) try: result = util.skopeo_inspect("docker://{}".format(fqdn), return_json=True) self._remote_inspect = result return fqdn except ValueError as e: - util.write_err("Failed: {}".format(e)) + if not quiet: + util.write_err("Failed: {}".format(e)) continue raise RegistryInspectError("Unable to resolve {}".format(self.orig_input)) diff --git a/Atomic/images.py b/Atomic/images.py index d0c8202..eaf9eb7 100644 --- a/Atomic/images.py +++ b/Atomic/images.py @@ -49,7 +49,7 @@ def cli(subparser): action="store_true", help=_("Delete image from remote repository")) - delete_parser.add_argument("--storage", default=storage, dest="storage", + delete_parser.add_argument("--storage", default=None, dest="storage", help=_("Specify the storage from which to delete the image from. " "If not specified and there are images with the same name in " "different storages, you will be prompted to specify.")) @@ -137,6 +137,7 @@ class Images(Atomic): if self.args.debug: util.write_out(str(self.args)) + self.be_utils.dump_backends() _images = self._get_images() for i in _images: diff --git a/Atomic/info.py b/Atomic/info.py index e0058a0..a26f95b 100644 --- a/Atomic/info.py +++ b/Atomic/info.py @@ -34,7 +34,7 @@ def cli(subparser, hidden=False): infop.add_argument("--remote", dest="force", action='store_true', default=False, help=_('ignore local images and only scan registries')) - infop.add_argument("--storage", default=storage, dest="storage", + infop.add_argument("--storage", default=None, dest="storage", help=_("Specify the storage of the image. " "If not specified and there are images with the same name in " "different storages, you will be prompted to specify.")) @@ -53,7 +53,7 @@ def cli_version(subparser, hidden=False): versionp.add_argument("-r", "--recurse", default=False, dest="recurse", action="store_true", help=_("recurse through all layers")) - versionp.add_argument("--storage", default=storage, dest="storage", + versionp.add_argument("--storage", default=None, dest="storage", help=_("Specify the storage of the image. " "If not specified and there are images with the same name in " "different storages, you will be prompted to specify.")) @@ -87,7 +87,7 @@ class Info(Atomic): write_func("") def get_layer_objects(self): - _, img_obj = self.beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage) + _, img_obj = self.beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) return img_obj.layers def dbus_version(self): @@ -117,7 +117,7 @@ class Info(Atomic): img_obj = be.make_remote_image(self.image) else: # The image is local - be, img_obj = self.beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage) + be, img_obj = self.beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) with closing(StringIO()) as buf: try: @@ -125,7 +125,8 @@ class Info(Atomic): except RegistryInspectError: info_name = img_obj.input_name buf.write("Image Name: {}\n".format(info_name)) - buf.writelines(sorted(["{}: {}\n".format(k, v) for k,v in list(img_obj.labels.items())])) + if img_obj.labels: + buf.writelines(sorted(["{}: {}\n".format(k, v) for k,v in list(img_obj.labels.items())])) if img_obj.template_variables_set: buf.write("\n\nTemplate variables with default value, but overridable with --set:\n") buf.writelines(["{}: {}\n".format(k, v) for k,v in diff --git a/Atomic/objects/image.py b/Atomic/objects/image.py index 4b3de46..681d0cb 100644 --- a/Atomic/objects/image.py +++ b/Atomic/objects/image.py @@ -117,7 +117,7 @@ class Image(object): if not self.registry: ri = RegistryInspect(registry=self.registry, repo=self.repo, image=self.image, tag=self.tag, orig_input=self.input_name) - self._fq_name = ri.find_image_on_registry() + self._fq_name = ri.find_image_on_registry(quiet=True) propagate(self._fq_name) return self._fq_name diff --git a/Atomic/run.py b/Atomic/run.py index 059aa21..57e4706 100644 --- a/Atomic/run.py +++ b/Atomic/run.py @@ -89,7 +89,7 @@ class Run(Atomic): db.pull_image(self.image) img_object = db.has_image(self.image) except RegistryInspectError: - util.write_err("Unable to find image {}".format(self.image)) + raise ValueError("Unable to find image {}".format(self.image)) db.run(img_object, atomic=self, args=self.args) diff --git a/Atomic/syscontainers.py b/Atomic/syscontainers.py index fe4b959..2f5ed24 100644 --- a/Atomic/syscontainers.py +++ b/Atomic/syscontainers.py @@ -84,6 +84,10 @@ class SystemContainers(object): stdout=DEVNULL, stderr=DEVNULL) + @property + def available(self): + return OSTREE_PRESENT + def _checkout_layer(self, repo, rootfs_fd, rootfs, rev): # ostree 2016.8 has a glib introspection safe API for checkout, use it # when available. diff --git a/Atomic/update.py b/Atomic/update.py index 461ae05..1e28467 100644 --- a/Atomic/update.py +++ b/Atomic/update.py @@ -25,7 +25,7 @@ def cli(subparser, hidden=False): updatep.add_argument("-f", "--force", default=False, dest="force", action="store_true", help=_("remove all containers based on this image")) - updatep.add_argument("--storage", default=storage, dest="storage", + updatep.add_argument("--storage", default=None, dest="storage", help=_("Specify the storage of the image. Defaults to: %s" % storage)) updatep.add_argument("image", help=_("container image")) @@ -38,7 +38,7 @@ class Update(Atomic): write_out(str(self.args)) beu = BackendUtils() try: - be, img_obj = beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage) + be, img_obj = beu.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) input_name = img_obj.input_name except ValueError: raise ValueError("{} not found locally. Unable to update".format(self.image)) diff --git a/Atomic/util.py b/Atomic/util.py index 4dde585..d85ec81 100644 --- a/Atomic/util.py +++ b/Atomic/util.py @@ -57,7 +57,10 @@ input, is_python2 = check_if_python2() # pylint: disable=redefined-builtin def get_registries(): registries = [] with AtomicDocker() as c: - dconf = c.info() + try: + dconf = c.info() + except requests.exceptions.ConnectionError: + raise ValueError("This Atomic function requires an active docker daemon.") search_regs = [x['Name'] for x in dconf['Registries']] rconf = dconf['RegistryConfig']['IndexConfigs'] # docker.io is special diff --git a/Atomic/verify.py b/Atomic/verify.py index 744c12f..5c04feb 100644 --- a/Atomic/verify.py +++ b/Atomic/verify.py @@ -33,7 +33,7 @@ def cli(subparser, hidden=False): verifyp.add_argument("--no-validate", default=False, dest="no_validate", action="store_true", help=_("disable validating system images")) - verifyp.add_argument("--storage", default=storage, dest="storage", + verifyp.add_argument("--storage", default=None, dest="storage", help=_("Specify the storage of the image. " "If not specified and there are images with the same name in " "different storages, you will be prompted to specify.")) @@ -86,7 +86,7 @@ class Verify(Atomic): return layers def _verify(self): - be, img_obj = self.backend_utils.get_backend_and_image_obj(self.image, self.args.storage) + be, img_obj = self.backend_utils.get_backend_and_image_obj(self.image, str_preferred_backend=self.args.storage or storage, required=True if self.args.storage else False) remote_img_name = "{}:latest".format(util.Decompose(img_obj.fq_name).no_tag) remote_img_obj = be.make_remote_image(remote_img_name) return img_obj.layers, remote_img_obj.layers diff --git a/bash/atomic b/bash/atomic index 8d3f018..33ce2e2 100644 --- a/bash/atomic +++ b/bash/atomic @@ -483,6 +483,7 @@ _atomic_containers_delete() { local all_options="$options_with_args --all -a --force -f + --storage --help "