Initial commit of crio-lxc

Use an init wrapper to bind in the syncfifo over which we will then inform cri-o that we are ready. This way we don't need any coordination from liblxc at all to emulate the desired protocol. Signed-off-by: Michael McCracken <mikmccra@cisco.com> Signed-off-by: Tycho Andersen <tycho@tycho.ws> Signed-off-by: Serge Hallyn <shallyn@cisco.com>
2026-02-05 09:45:04 +01:00 · 2019-04-04 20:46:34 -07:00
parent 77276bea1c
commit 19cd17cd88
16 changed files with 1143 additions and 25 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1 @@
+*~
--- a/26
+++ b/26
@@ -1,3 +1,4 @@
+
                                 Apache License
                           Version 2.0, January 2004
                        http://www.apache.org/licenses/
@@ -174,28 +175,3 @@
      of your accepting any such warranty or additional liability.

   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--- a/22
+++ b/22
@@ -0,0 +1,22 @@
+GO_SRC=$(shell find . -name \*.go)
+COMMIT_HASH=$(shell git rev-parse HEAD)
+COMMIT=$(if $(shell git status --porcelain --untracked-files=no),$(COMMIT_HASH)-dirty,$(COMMIT_HASH))
+TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats))
+
+crio-lxc: $(GO_SRC)
+	go build -ldflags "-X main.version=$(COMMIT)" -o crio-lxc ./cmd
+
+# make test TEST=basic will run only the basic test.
+.PHONY: check
+check: crio-lxc
+	go fmt ./... && ([ -z $(TRAVIS) ] || git diff --quiet)
+	go test ./...
+	sudo -E "PATH=$$PATH" bats -t $(patsubst %,test/%.bats,$(TEST))
+
+.PHONY: vendorup
+vendorup:
+	go get -u
+
+.PHONY: clean
+clean:
+	-rm -r crio-lxc
--- a/README.md
+++ b/README.md
@@ -0,0 +1,14 @@
+# crio-lxc
+
+This is a wrapper around [LXC](https://github.com/lxc/lxc) which can be used as
+a drop-in container runtime replacement for use by
+[CRI-O](https://github.com/kubernetes-sigs/cri-o).
+
+To use this, simply build it:
+
+```
+make
+```
+
+Then specify the `crio-lxc` binary you just built as the value for
+`default_runtime` in the `crio.runtime` section of `/etc/crio/crio.conf`.
--- a/cmd/create.go
+++ b/cmd/create.go
@@ -0,0 +1,212 @@
+package main
+
+import (
+	"fmt"
+	"golang.org/x/sys/unix"
+
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/apex/log"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var createCmd = cli.Command{
+	Name:      "create",
+	Usage:     "create a container from a bundle directory",
+	ArgsUsage: "<containerID>",
+	Action:    doCreate,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "bundle",
+			Usage: "set bundle directory",
+			Value: ".",
+		},
+		cli.IntFlag{
+			Name:  "console-socket",
+			Usage: "pty master FD", // TODO not handled yet
+		},
+		cli.StringFlag{
+			Name:  "pid-file",
+			Usage: "path to write container PID", // TODO not handled yet
+		},
+	},
+}
+
+func emitFifoWaiter(file string) error {
+	fifoWaiter := fmt.Sprintf(`#!/bin/sh
+stat /syncfifo
+echo "%s" | tee /syncfifo
+exec $@
+`, SYNC_FIFO_CONTENT)
+
+	return ioutil.WriteFile(file, []byte(fifoWaiter), 0755)
+}
+
+func doCreate(ctx *cli.Context) error {
+	containerID := ctx.Args().Get(0)
+	if len(containerID) == 0 {
+		fmt.Fprintf(os.Stderr, "missing container ID\n")
+		cli.ShowCommandHelpAndExit(ctx, "create", 1)
+	}
+	log.Infof("creating container %s", containerID)
+
+	exists, err := containerExists(containerID)
+	if err != nil {
+		return errors.Wrap(err, "failed to check if container exists")
+	}
+	if exists {
+		return fmt.Errorf("container '%s' already exists", containerID)
+	}
+
+	c, err := lxc.NewContainer(containerID, LXC_PATH)
+	if err != nil {
+		return errors.Wrap(err, "failed to create new container")
+	}
+	defer c.Release()
+
+	spec, err := readBundleSpec(filepath.Join(ctx.String("bundle"), "config.json"))
+	if err != nil {
+		return errors.Wrap(err, "couldn't load bundle spec")
+	}
+
+	if err := os.MkdirAll(filepath.Join(LXC_PATH, containerID), 0770); err != nil {
+		return errors.Wrap(err, "failed to create container dir")
+	}
+
+	if err := makeSyncFifo(filepath.Join(LXC_PATH, containerID)); err != nil {
+		return errors.Wrap(err, "failed to make sync fifo")
+	}
+
+	if err := configureContainer(ctx, c, spec); err != nil {
+		return errors.Wrap(err, "failed to configure container")
+	}
+
+	log.Infof("created syncfifo, executing %#v", spec.Process.Args)
+
+	if err := startContainer(c, spec); err != nil {
+		return errors.Wrap(err, "failed to start the container init")
+	}
+
+	log.Infof("created container %s in lxcdir %s", containerID, LXC_PATH)
+	return nil
+}
+
+func configureContainer(ctx *cli.Context, c *lxc.Container, spec *specs.Spec) error {
+	if ctx.Bool("debug") {
+		c.SetVerbosity(lxc.Verbose)
+	}
+
+	if err := configureLogging(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to configure logging")
+	}
+
+	// rootfs
+	// todo Root.Readonly? - use lxc.rootfs.options
+	if err := c.SetConfigItem("lxc.rootfs.path", spec.Root.Path); err != nil {
+		return errors.Wrapf(err, "failed to set rootfs: '%s'", spec.Root.Path)
+	}
+	if err := c.SetConfigItem("lxc.rootfs.managed", "0"); err != nil {
+		return errors.Wrap(err, "failed to set rootfs.managed to 0")
+	}
+
+	for _, envVar := range spec.Process.Env {
+		if err := c.SetConfigItem("lxc.environment", envVar); err != nil {
+			return fmt.Errorf("error setting environment variable '%s': %v", envVar, err)
+		}
+	}
+
+	for _, ms := range spec.Mounts {
+		opts := strings.Join(ms.Options, ",")
+		mnt := fmt.Sprintf("%s %s %s %s", ms.Source, ms.Destination, ms.Type, opts)
+		if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
+			return errors.Wrap(err, "failed to set mount config")
+		}
+	}
+
+	mnt := fmt.Sprintf("%s %s none ro,bind,create=file", path.Join(LXC_PATH, c.Name(), SYNC_FIFO_PATH), strings.Trim(SYNC_FIFO_PATH, "/"))
+	if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
+		return errors.Wrap(err, "failed to set syncfifo mount config entry")
+	}
+
+	err := emitFifoWaiter(path.Join(spec.Root.Path, "fifo-wait"))
+	if err != nil {
+		return errors.Wrapf(err, "couldn't write wrapper init")
+	}
+
+	if err := c.SetConfigItem("lxc.init.cwd", spec.Process.Cwd); err != nil {
+		return errors.Wrap(err, "failed to set CWD")
+	}
+
+	if err := c.SetConfigItem("lxc.uts.name", spec.Hostname); err != nil {
+		return errors.Wrap(err, "failed to set hostname")
+	}
+
+	argsString := "/fifo-wait " + strings.Join(spec.Process.Args, " ")
+	if err := c.SetConfigItem("lxc.execute.cmd", argsString); err != nil {
+		return errors.Wrap(err, "failed to set lxc.execute.cmd")
+
+	}
+	if err := c.SetConfigItem("lxc.hook.version", "1"); err != nil {
+		return errors.Wrap(err, "failed to set hook version")
+	}
+
+	// capabilities?
+
+	// if !spec.Process.Terminal {
+	// 	passFdsToContainer()
+	// }
+
+	// Write out final config file for debugging and use with lxc-attach:
+	// Do not edit config after this.
+	savedConfigFile := filepath.Join(LXC_PATH, c.Name(), "config")
+	if err := c.SaveConfigFile(savedConfigFile); err != nil {
+		return errors.Wrapf(err, "failed to save config file to '%s'", savedConfigFile)
+	}
+
+	return nil
+}
+
+func makeSyncFifo(dir string) error {
+	fifoFilename := filepath.Join(dir, "syncfifo")
+	prevMask := unix.Umask(0000)
+	defer unix.Umask(prevMask)
+	if err := unix.Mkfifo(fifoFilename, 0622); err != nil {
+		return errors.Wrapf(err, "failed to make fifo '%s'", fifoFilename)
+	}
+	return nil
+}
+
+func startContainer(c *lxc.Container, spec *specs.Spec) error {
+	binary, err := os.Readlink("/proc/self/exe")
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(
+		binary,
+		"internal",
+		c.Name(),
+		spec.Root.Path,
+		filepath.Join(LXC_PATH, c.Name(), "config"),
+	)
+
+	if !spec.Process.Terminal {
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+	}
+
+	cmdErr := cmd.Start()
+
+	return cmdErr
+
+}
--- a/cmd/defs.go
+++ b/cmd/defs.go
@@ -0,0 +1,7 @@
+package main
+
+var (
+	CURRENT_OCI_VERSION = "0.2.1"
+	// controlled by --lxc-path arg to main
+	LXC_PATH = ""
+)
--- a/cmd/delete.go
+++ b/cmd/delete.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var deleteCmd = cli.Command{
+	Name:   "delete",
+	Usage:  "deletes a container",
+	Action: doDelete,
+	ArgsUsage: `[containerID]
+
+<containerID> is the ID of the container to delete
+`,
+	Flags: []cli.Flag{},
+}
+
+func doDelete(ctx *cli.Context) error {
+	containerID := ctx.Args().Get(0)
+	if len(containerID) == 0 {
+		fmt.Fprintf(os.Stderr, "missing container ID\n")
+		cli.ShowCommandHelpAndExit(ctx, "state", 1)
+	}
+
+	exists, err := containerExists(containerID)
+	if err != nil {
+		return errors.Wrap(err, "failed to check if container exists")
+	}
+	if !exists {
+		return fmt.Errorf("container '%s' not found", containerID)
+	}
+
+	c, err := lxc.NewContainer(containerID, LXC_PATH)
+	if err != nil {
+		return errors.Wrap(err, "failed to load container")
+	}
+	defer c.Release()
+
+	if err := configureLogging(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to configure logging")
+
+	}
+
+	if c.Running() {
+		return fmt.Errorf("container '%s' is running, cannot delete.", containerID)
+	}
+
+	// TODO: lxc-destroy deletes the rootfs.
+	// this appears to contradict the runtime spec:
+
+	// "Note that resources associated with the container,
+	// but not created by this container, MUST NOT be deleted.Note
+	// that resources associated with the container, but not
+	// created by this container, MUST NOT be deleted.
+
+	if err := c.Destroy(); err != nil {
+		return errors.Wrap(err, "failed to delete container.")
+	}
+
+	// TODO - because we set rootfs.managed=0, Destroy() doesn't
+	// delete the /var/lib/lxc/$containerID/config file:
+	configDir := filepath.Join(LXC_PATH, containerID)
+	if err := os.RemoveAll(configDir); err != nil {
+		return errors.Wrapf(err, "failed to remove %s", configDir)
+	}
+
+	return nil
+}
--- a/cmd/fifo-wait.go
+++ b/cmd/fifo-wait.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+)
+
+const (
+	SYNC_FIFO_PATH    = "/syncfifo"
+	SYNC_FIFO_CONTENT = "meshuggah rocks"
+)
+
+var fifoWaitCmd = cli.Command{
+	Name:   "fifo-wait",
+	Action: doFifoWait,
+	Hidden: true,
+}
+
+func doFifoWait(ctx *cli.Context) error {
+	syncPipe, err := os.Open(SYNC_FIFO_PATH)
+	if err != nil {
+		return errors.Wrapf(err, "couldn't open %s", SYNC_FIFO_PATH)
+	}
+	defer syncPipe.Close()
+
+	fi, err := syncPipe.Stat()
+	if err != nil {
+		return errors.Wrapf(err, "couldn't stat %s", SYNC_FIFO_PATH)
+	}
+
+	if fi.Mode()&os.ModeNamedPipe == 0 {
+		return errors.Errorf("%s exists and is not a named pipe", SYNC_FIFO_PATH)
+	}
+
+	_, err = syncPipe.WriteString(SYNC_FIFO_CONTENT)
+	if err != nil {
+		return errors.Wrapf(err, "could write to fifo")
+	}
+
+	cmd := exec.Command(ctx.Args().Get(0), ctx.Args().Tail()...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
--- a/cmd/internal.go
+++ b/cmd/internal.go
@@ -0,0 +1,117 @@
+/*
+ * This file is a little bit strange. The problem is that we want to do
+ * daemonized containers with liblxc, but we can't spawn containers in threaded
+ * environments (i.e. golang), with go-lxc. So instead, we embed some C into
+ * our program that catches execution before golang starts. This way, we can do
+ * a tiny C program to actually spawn the container.
+ *
+ */
+package main
+
+// #cgo LDFLAGS: -llxc
+/*
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <signal.h>
+
+#include <lxc/lxccontainer.h>
+
+static int spawn_container(char *name, char *lxcpath, char *config)
+{
+	struct lxc_container *c;
+
+	c = lxc_container_new(name, lxcpath);
+	if (!c) {
+		fprintf(stderr, "failed to create container %s\n", name);
+		return -1;
+	}
+
+	c->clear_config(c);
+	if (!c->load_config(c, config)) {
+		fprintf(stderr, "failed to load container config at %s\n", config);
+		return -1;
+	}
+
+	c->daemonize = false;
+	if (!c->start(c, 1, NULL)) {
+		fprintf(stderr, "failed to start container %s\n", name);
+		return -1;
+	}
+
+	return c->error_num;
+}
+
+// main function for the "internal" command. Right now, arguments look like:
+// argv[0] internal <container_name> <lxcpath> <config_path>
+__attribute__((constructor)) void internal(void)
+{
+	int ret, status;
+	char buf[4096];
+	ssize_t size;
+	char *cur, *name, *lxcpath, *config_path;
+
+	ret = open("/proc/self/cmdline", O_RDONLY);
+	if (ret < 0) {
+		perror("error: open");
+		exit(96);
+	}
+
+	if ((size = read(ret, buf, sizeof(buf)-1)) < 0) {
+		close(ret);
+		perror("error: read");
+		exit(96);
+	}
+	close(ret);
+
+	// /proc/self/cmdline is null separated, but let's be real safe
+	buf[size] = 0;
+	cur = buf;
+
+#define ADVANCE_ARG		\
+	do {			\
+		while (*cur) {	\
+			cur++;	\
+		}		\
+		cur++;		\
+	} while (0)
+
+	// skip argv[0]
+	ADVANCE_ARG;
+
+	// is this really the internal command, if not, continue normal execution
+	if (strcmp(cur, "internal"))
+		return;
+
+	ADVANCE_ARG;
+	name = cur;
+	ADVANCE_ARG;
+	lxcpath = cur;
+	ADVANCE_ARG;
+	config_path = cur;
+
+	ret = isatty(STDIN_FILENO);
+	if (ret < 0) {
+		perror("isatty");
+		exit(96);
+	}
+
+	// If this is non interactive, get rid of our controlling terminal,
+	// since we don't want lxc's setting of ISIG to ignore user's ^Cs.
+	if (!ret)
+		setsid();
+
+	status = spawn_container(name, lxcpath, config_path);
+
+	// Try and propagate the container's exit code.
+	if (WIFEXITED(status)) {
+		exit(WEXITSTATUS(status));
+	} else {
+		kill(0, WTERMSIG(status));
+		exit(EXIT_FAILURE);
+	}
+}
+*/
+import "C"
--- a/cmd/kill.go
+++ b/cmd/kill.go
@@ -0,0 +1,106 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	//"github.com/apex/log"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var killCmd = cli.Command{
+	Name:   "kill",
+	Usage:  "sends a signal to a container",
+	Action: doKill,
+	ArgsUsage: `[containerID]
+
+<containerID> is the ID of the container to send a signal to
+`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "signal",
+			Usage: "the signal to send, as a string",
+			Value: "TERM",
+		},
+	},
+}
+var signalMap = map[string]syscall.Signal{
+	"ABRT":   unix.SIGABRT,
+	"ALRM":   unix.SIGALRM,
+	"BUS":    unix.SIGBUS,
+	"CHLD":   unix.SIGCHLD,
+	"CLD":    unix.SIGCLD,
+	"CONT":   unix.SIGCONT,
+	"FPE":    unix.SIGFPE,
+	"HUP":    unix.SIGHUP,
+	"ILL":    unix.SIGILL,
+	"INT":    unix.SIGINT,
+	"IO":     unix.SIGIO,
+	"IOT":    unix.SIGIOT,
+	"KILL":   unix.SIGKILL,
+	"PIPE":   unix.SIGPIPE,
+	"POLL":   unix.SIGPOLL,
+	"PROF":   unix.SIGPROF,
+	"PWR":    unix.SIGPWR,
+	"QUIT":   unix.SIGQUIT,
+	"SEGV":   unix.SIGSEGV,
+	"STKFLT": unix.SIGSTKFLT,
+	"STOP":   unix.SIGSTOP,
+	"SYS":    unix.SIGSYS,
+	"TERM":   unix.SIGTERM,
+	"TRAP":   unix.SIGTRAP,
+	"TSTP":   unix.SIGTSTP,
+	"TTIN":   unix.SIGTTIN,
+	"TTOU":   unix.SIGTTOU,
+	"URG":    unix.SIGURG,
+	"USR1":   unix.SIGUSR1,
+	"USR2":   unix.SIGUSR2,
+	"VTALRM": unix.SIGVTALRM,
+	"WINCH":  unix.SIGWINCH,
+	"XCPU":   unix.SIGXCPU,
+	"XFSZ":   unix.SIGXFSZ,
+}
+
+func doKill(ctx *cli.Context) error {
+	containerID := ctx.Args().Get(0)
+	if len(containerID) == 0 {
+		fmt.Fprintf(os.Stderr, "missing container ID\n")
+		cli.ShowCommandHelpAndExit(ctx, "state", 1)
+	}
+
+	exists, err := containerExists(containerID)
+	if err != nil {
+		return errors.Wrap(err, "failed to check if container exists")
+	}
+	if !exists {
+		return fmt.Errorf("container '%s' not found", containerID)
+	}
+
+	c, err := lxc.NewContainer(containerID, LXC_PATH)
+	if err != nil {
+		return errors.Wrap(err, "failed to load container")
+	}
+	defer c.Release()
+
+	if err := configureLogging(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to configure logging")
+
+	}
+
+	if !c.Running() {
+		return fmt.Errorf("container '%s' is not running", containerID)
+	}
+
+	pid := c.InitPid()
+
+	if err := unix.Kill(pid, signalMap[ctx.String("signal")]); err != nil {
+		return errors.Wrap(err, "failed to send signal")
+	}
+	return nil
+}
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/apex/log"
+	"github.com/urfave/cli"
+)
+
+var (
+	version = ""
+	debug   = false
+)
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "crio-lxc"
+	app.Usage = "crio-lxc is a CRI compliant runtime wrapper for lxc"
+	app.Version = version
+	app.Commands = []cli.Command{
+		stateCmd,
+		createCmd,
+		startCmd,
+		killCmd,
+		deleteCmd,
+	}
+
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "debug",
+			Usage: "enable debug mode",
+		},
+		cli.StringFlag{
+			Name:  "log-level",
+			Usage: "set log level for LXC",
+		},
+		cli.StringFlag{
+			Name:  "log-file",
+			Usage: "log file for LXC",
+		},
+		cli.StringFlag{
+			Name:  "lxc-path, root",
+			Usage: "set the lxc path to use",
+			Value: "/var/lib/lxc",
+		},
+	}
+
+	app.Before = func(ctx *cli.Context) error {
+		LXC_PATH = ctx.String("lxc-path")
+
+		debug = ctx.Bool("debug")
+		return nil
+	}
+
+	log.SetLevel(log.InfoLevel)
+
+	if err := app.Run(os.Args); err != nil {
+		format := "error: %v\n"
+		if debug {
+			format = "error: %+v\n"
+		}
+
+		fmt.Fprintf(os.Stderr, format, err)
+		os.Exit(1)
+	}
+}
--- a/cmd/start.go
+++ b/cmd/start.go
@@ -0,0 +1,63 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/apex/log"
+	//	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var startCmd = cli.Command{
+	Name:   "start",
+	Usage:  "starts a container",
+	Action: doStart,
+	ArgsUsage: `[containerID]
+
+starts <containerID>
+`,
+}
+
+func doStart(ctx *cli.Context) error {
+	containerID := ctx.Args().Get(0)
+	if len(containerID) == 0 {
+		fmt.Fprintf(os.Stderr, "missing container ID\n")
+		cli.ShowCommandHelpAndExit(ctx, "state", 1)
+	}
+
+	log.Infof("about to create container")
+	c, err := lxc.NewContainer(containerID, LXC_PATH)
+	if err != nil {
+		return errors.Wrap(err, "failed to load container")
+	}
+	defer c.Release()
+	log.Infof("checking if running")
+	if c.Running() {
+		return fmt.Errorf("'%s' is already running", containerID)
+	}
+	log.Infof("not running, can start")
+	fifoPath := filepath.Join(LXC_PATH, containerID, "syncfifo")
+	log.Infof("opening fifo '%s'", fifoPath)
+	f, err := os.OpenFile(fifoPath, os.O_RDWR, 0)
+	if err != nil {
+		return errors.Wrap(err, "failed to open sync fifo")
+	}
+
+	log.Infof("opened fifo, reading")
+	data := make([]byte, len(SYNC_FIFO_CONTENT))
+	n, err := f.Read(data)
+	if err != nil {
+		return errors.Wrapf(err, "problem reading from fifo")
+	}
+	if n != len(SYNC_FIFO_CONTENT) || string(data) != SYNC_FIFO_CONTENT {
+		return errors.Errorf("bad fifo content: %s", string(data))
+	}
+
+	log.Infof("read '%s' from fifo, done", data)
+	return nil
+}
--- a/cmd/state.go
+++ b/cmd/state.go
@@ -0,0 +1,82 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	//	"github.com/apex/log"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+var stateCmd = cli.Command{
+	Name:   "state",
+	Usage:  "returns state of a container",
+	Action: doState,
+	ArgsUsage: `[containerID]
+
+<containerID> is the ID of the container you want to know about.
+`,
+	Flags: []cli.Flag{},
+}
+
+func doState(ctx *cli.Context) error {
+	containerID := ctx.Args().Get(0)
+	if len(containerID) == 0 {
+		fmt.Fprintf(os.Stderr, "missing container ID\n")
+		cli.ShowCommandHelpAndExit(ctx, "state", 1)
+	}
+
+	exists, err := containerExists(containerID)
+	if err != nil {
+		return errors.Wrap(err, "failed to check if container exists")
+	}
+	if !exists {
+		return fmt.Errorf("container '%s' not found", containerID)
+	}
+
+	c, err := lxc.NewContainer(containerID, LXC_PATH)
+	if err != nil {
+		return errors.Wrap(err, "failed to load container")
+	}
+	defer c.Release()
+
+	if err := configureLogging(ctx, c); err != nil {
+		return errors.Wrap(err, "failed to configure logging")
+
+	}
+
+	// TODO need to detect 'created' per
+	// https://github.com/opencontainers/runtime-spec/blob/v1.0.0-rc4/runtime.md#state
+	// it means "the container process has neither exited nor executed the user-specified program"
+	status := "stopped"
+	if c.Running() {
+		status = "running"
+	}
+	pid := 0
+	// bundlePath is the enclosing directory of the rootfs:
+	// https://github.com/opencontainers/runtime-spec/blob/v1.0.0-rc4/bundle.md
+	bundlePath := filepath.Dir(c.ConfigItem("lxc.rootfs.path")[0])
+	annotations := map[string]string{}
+	s := specs.State{
+		Version:     CURRENT_OCI_VERSION,
+		ID:          containerID,
+		Status:      status,
+		Pid:         pid,
+		Bundle:      bundlePath,
+		Annotations: annotations,
+	}
+
+	stateJson, err := json.Marshal(s)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal json")
+	}
+	fmt.Fprintf(os.Stdout, string(stateJson))
+
+	return nil
+}
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	lxc "gopkg.in/lxc/go-lxc.v2"
+)
+
+func readBundleSpec(specFilePath string) (spec *specs.Spec, err error) {
+	specFile, err := os.Open(specFilePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open spec file '%s'", specFilePath)
+	}
+	defer specFile.Close()
+	err = json.NewDecoder(specFile).Decode(&spec)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to decode spec file")
+	}
+
+	return spec, nil
+}
+
+func configureLogging(ctx *cli.Context, c *lxc.Container) error {
+	if ctx.GlobalIsSet("log-level") {
+		logLevel := lxc.TRACE
+		switch ctx.GlobalString("log-level") {
+		case "trace":
+			logLevel = lxc.TRACE
+		case "debug":
+			logLevel = lxc.DEBUG
+		case "info":
+			logLevel = lxc.INFO
+		case "warn":
+			logLevel = lxc.WARN
+		case "", "error":
+			logLevel = lxc.ERROR
+		default:
+			return fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error")
+		}
+		c.SetLogLevel(logLevel)
+	}
+
+	if ctx.GlobalIsSet("log-file") {
+		c.SetLogFile(ctx.GlobalString("log-file"))
+	}
+	return nil
+}
+
+func pathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return true, err
+}
+
+func containerExists(containerID string) (bool, error) {
+	// check for container existence by looking for config file.
+	// otherwise NewContainer will return an empty container
+	// struct and we'll report wrong info
+	configExists, err := pathExists(filepath.Join(LXC_PATH, containerID, "config"))
+	if err != nil {
+		return false, errors.Wrap(err, "failed to check path existence of config")
+	}
+
+	return configExists, nil
+}
--- a/go.mod
+++ b/go.mod
@@ -0,0 +1,28 @@
+module github.com/lxc/crio-lxc
+
+require (
+	github.com/anuvu/stacker v0.4.0
+	github.com/apex/log v1.1.0
+	github.com/gorilla/websocket v1.4.0 // indirect
+	github.com/juju/loggo v0.0.0-20190212223446-d976af380377 // indirect
+	github.com/lxc/lxd v0.0.0-20190404234020-f51c28a37443
+	github.com/openSUSE/umoci v0.4.4
+	github.com/opencontainers/runtime-spec v1.0.1
+	github.com/opencontainers/selinux v1.2.1 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/rogpeppe/godef v1.1.1 // indirect
+	github.com/sirupsen/logrus v1.4.1 // indirect
+	github.com/urfave/cli v1.20.0
+	golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 // indirect
+	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect
+	golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67
+	golang.org/x/tools v0.0.0-20190405180640-052fc3cfdbc2 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/lxc/go-lxc.v2 v2.0.0-20190324192716-2f350e4a2980
+	gopkg.in/yaml.v2 v2.2.2
+	k8s.io/client-go v11.0.0+incompatible // indirect
+)
+
+replace github.com/vbatts/go-mtree v0.4.4 => github.com/vbatts/go-mtree v0.4.5-0.20190122034725-8b6de6073c1a
+
+replace github.com/openSUSE/umoci v0.4.4 => github.com/tych0/umoci v0.1.1-0.20190402232331-556620754fb1
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,223 @@
+9fans.net/go v0.0.0-20181112161441-237454027057 h1:OcHlKWkAMJEF1ndWLGxp5dnJQkYM/YImUOvsBoz6h5E=
+9fans.net/go v0.0.0-20181112161441-237454027057/go.mod h1:diCsxrliIURU9xsYtjCp5AbpQKqdhKmf0ujWDUSkfoY=
+code.cloudfoundry.org/systemcerts v0.0.0-20180917154049-ca00b2f806f2/go.mod h1:EXawaFLz9fhxAjRCeaqP+Wr5t5+h75UzbHEA7ybrr1o=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Microsoft/go-winio v0.0.0-20190117211522-75bf6ca3d7cb/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/anuvu/stacker v0.4.0 h1:+r8MQr8RVlFuVe9W7mxZ/pF/SBtX5XIvaBOcmkPqlI0=
+github.com/anuvu/stacker v0.4.0/go.mod h1:RGdZIvdQLCxFPND+8jXVppAUPbRS/dkDWe65qt274Gg=
+github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI=
+github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/boltdb/bolt v0.0.0-20180302180052-fd01fc79c553 h1:yvSJ8qbaWLeS7COhu2KJ0epn4mmc+aGeBP7Dpg7xQTY=
+github.com/boltdb/bolt v0.0.0-20180302180052-fd01fc79c553/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/cheggaaa/pb v1.0.27 h1:wIkZHkNfC7R6GI5w7l/PdAdzXzlrbcI3p8OAlnkTsnc=
+github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
+github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M=
+github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containers/image v0.0.0-20190208010805-4629bcc4825f h1:Z7kajYzNuO4jIjGfQSgF88QQZL8uBlGggRcukmldxow=
+github.com/containers/image v0.0.0-20190208010805-4629bcc4825f/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M=
+github.com/containers/storage v0.0.0-20190207215558-06b6c2e4cf25/go.mod h1:+RirK6VQAqskQlaTBrOG6ulDvn4si2QjFE1NZCn06MM=
+github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docker/distribution v0.0.0-20190205005809-0d3efadf0154 h1:C8WBRZDiZn3IZnBlbHVeTWF32XhVGK69Li4GC/3jL9Q=
+github.com/docker/distribution v0.0.0-20190205005809-0d3efadf0154/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v0.0.0-20190207111444-e6fe7f8f2936 h1:eRJM2oxktfS3HP/ziIJuvn9QXxq1rP/2mNJj04uXklQ=
+github.com/docker/docker v0.0.0-20190207111444-e6fe7f8f2936/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.0.0-20180925085122-123ba1b7cd64 h1:imB4LnBHHEccRIBjH4lY4X3TC8FPRPue/nkCiJtLERM=
+github.com/docker/docker-credential-helpers v0.0.0-20180925085122-123ba1b7cd64/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
+github.com/docker/go-connections v0.0.0-20180821093606-97c2040d34df h1:ADMjlaDGEn0OOQIieyxanhAt41jcngf8rf78X2eKNLw=
+github.com/docker/go-connections v0.0.0-20180821093606-97c2040d34df/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA=
+github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/flosch/pongo2 v0.0.0-20181225140029-79872a7b2769 h1:XToLChWPMXLomJ2InnkrmUkddaXfevrmomMTFL+MaKU=
+github.com/flosch/pongo2 v0.0.0-20181225140029-79872a7b2769/go.mod h1:tbAXHifHQWNSpWbiJHpJTZH5fi3XHhDMdP//vuz9WS4=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/freddierice/go-losetup v0.0.0-20170407175016-fc9adea44124 h1:TVfi5xMshZAXzVXozESk8bi0JSTPwHkx7qtLOkkcu/c=
+github.com/freddierice/go-losetup v0.0.0-20170407175016-fc9adea44124/go.mod h1:zAk7fcFx45euzK9Az14j6Hd9n8Cwhnjp/NBfhSIAmFg=
+github.com/ghodss/yaml v0.0.0-20190206175653-d4115522f0fe h1:X2U2232O/PUczwdCr6+ScoThLvdBKg5yIu6hLcHFR88=
+github.com/ghodss/yaml v0.0.0-20190206175653-d4115522f0fe/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-check/check v1.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
+github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v0.0.0-20190205004414-7c8e298727d1/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/errors v0.0.0-20190207033735-e65537c515d7 h1:dMIPRDg6gi7CUp0Kj2+HxqJ5kTr1iAdzsXYIrLCNSmU=
+github.com/juju/errors v0.0.0-20190207033735-e65537c515d7/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/juju/loggo v0.0.0-20190212223446-d976af380377/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
+github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/crc32 v1.2.0/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
+github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=
+github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lxc/lxd v0.0.0-20190208124523-fe0844d45b32/go.mod h1:2BaZflfwsv8a3uy3/Vw+de4Avn4DSrAiqaHJjCIXMV4=
+github.com/lxc/lxd v0.0.0-20190404234020-f51c28a37443 h1:8VLxh92A974SkZlwBlj38PJprNFwNu3J7CRoTjOjsq4=
+github.com/lxc/lxd v0.0.0-20190404234020-f51c28a37443/go.mod h1:2BaZflfwsv8a3uy3/Vw+de4Avn4DSrAiqaHJjCIXMV4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.0-20181218000649-703b5e6b11ae h1:84Sj4V9L4dE13ryw5Vn/sxmxg5ebqhpIfTHflC5Gqrg=
+github.com/mattn/go-runewidth v0.0.0-20181218000649-703b5e6b11ae/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
+github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
+github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c h1:xa+eQWKuJ9MbB9FBL/eoNvDFvveAkz2LQoz8PzX7Q/4=
+github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c/go.mod h1:GhAqVMEWnTcW2dxoD/SO3n2enrgWl3y6Dnx4m59GvcA=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190208075259-dd023c457d84 h1:fDcjHkdDVppN5Fk3yoNtK13eYg8UWLVSzlr4Z8O8MQA=
+github.com/opencontainers/runc v0.0.0-20190208075259-dd023c457d84/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runtime-spec v1.0.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.1 h1:wY4pOY8fBdSIvs9+IDHC55thBuEulhzfSgKeC1yFvzQ=
+github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.7.0 h1:MIjqgwi4ZC+eVNGiYotCUYuTfs/oWDEcigK9Ra5ruHU=
+github.com/opencontainers/runtime-tools v0.7.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/selinux v1.0.0/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
+github.com/opencontainers/selinux v1.2.1/go.mod h1:+BLncwf63G4dgOzykXAxcmnFlUaOlkDdmw/CqsW6pjs=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1 h1:K47Rk0v/fkEfwfQet2KWhscE0cJzjgCCDBG2KHZoVno=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU=
+github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190208162519-de1b801bf34b h1:SSKVZGyOykopyi4bOAsGmbuEHPExRa+PEI2knZCALcU=
+github.com/prometheus/procfs v0.0.0-20190208162519-de1b801bf34b/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/rogpeppe/godef v1.1.1 h1:NujOtt9q9vIClRTB3sCZpavac+NMRaIayzrcz1h4fSE=
+github.com/rogpeppe/godef v1.1.1/go.mod h1:oEo1eMy1VUEHUzUIX4F7IqvMJRiz9UId44mvnR8oPlQ=
+github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4=
+github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8=
+github.com/sergi/go-diff v0.0.0-20180205163309-da645544ed44 h1:UoZTeCJuGZpwXXqamtSHypwjqpyGdR9smB5iMleBDJ8=
+github.com/sergi/go-diff v0.0.0-20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/tych0/umoci v0.1.1-0.20190402232331-556620754fb1 h1:o4wA23uJTjELWzDPI9c5fWshoXr5iMY8gbt0Vz2bM9Y=
+github.com/tych0/umoci v0.1.1-0.20190402232331-556620754fb1/go.mod h1:r5Md+fkaQUURwsULEptlOz0Vv2QlzbdbfHi+eBkk2uU=
+github.com/udhos/equalfile v0.3.0 h1:KhG4xhhkittrgIV/ekHtpEPh7MLxtbjm6kLEwp5Dlbg=
+github.com/udhos/equalfile v0.3.0/go.mod h1:1LOX9HjdFMke7ryP3IPby09FkswyY5KzhhsT37wLz/Y=
+github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok=
+github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/vbatts/go-mtree v0.4.5-0.20190122034725-8b6de6073c1a h1:nuTbu0QJphHUZMSgXx0U5a9+lhM9Mz6eLt/0gGEPgs4=
+github.com/vbatts/go-mtree v0.4.5-0.20190122034725-8b6de6073c1a/go.mod h1:3sazBqLG4bZYmgRTgdh9X3iKTzwBpp5CrREJDzrNSXY=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v0.0.0-20180719132039-b84684d0e066/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
+github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE=
+golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998=
+golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20181130195746-895048a75ecf/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190405180640-052fc3cfdbc2 h1:rlaAa9eBBj6AI2C90gKs2Q/XF6YFbBDpGSX+npdfPlk=
+golang.org/x/tools v0.0.0-20190405180640-052fc3cfdbc2/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo=
+gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/lxc/go-lxc.v2 v2.0.0-20181227225324-7c910f8a5edc/go.mod h1:4K0lbUXeslpmjwJZyW1lI6s5j97mrsj4+kpYwwvuLXo=
+gopkg.in/lxc/go-lxc.v2 v2.0.0-20190324192716-2f350e4a2980 h1:JH0hbXFbkeENYCUaso0BKGGuw5RyisZwJKYG4KuzzC4=
+gopkg.in/lxc/go-lxc.v2 v2.0.0-20190324192716-2f350e4a2980/go.mod h1:4K0lbUXeslpmjwJZyW1lI6s5j97mrsj4+kpYwwvuLXo=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=