incus-benchmark: Move to cmd/incus-benchmark

Signed-off-by: Stéphane Graber <stgraber@stgraber.org>

cmd/incus-benchmark/benchmark.go

@@ -0,0 +1,285 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/cyphar/incus/client"
+	"github.com/cyphar/incus/inc/config"
+	"github.com/cyphar/incus/shared/api"
+	"github.com/cyphar/incus/shared/version"
+)
+
+const userConfigKey = "user.lxd-benchmark"
+
+// PrintServerInfo prints out information about the server.
+func PrintServerInfo(c incus.InstanceServer) error {
+	server, _, err := c.GetServer()
+	if err != nil {
+		return err
+	}
+
+	env := server.Environment
+	fmt.Println("Test environment:")
+	fmt.Println("  Server backend:", env.Server)
+	fmt.Println("  Server version:", env.ServerVersion)
+	fmt.Println("  Kernel:", env.Kernel)
+	fmt.Println("  Kernel tecture:", env.KernelArchitecture)
+	fmt.Println("  Kernel version:", env.KernelVersion)
+	fmt.Println("  Storage backend:", env.Storage)
+	fmt.Println("  Storage version:", env.StorageVersion)
+	fmt.Println("  Container backend:", env.Driver)
+	fmt.Println("  Container version:", env.DriverVersion)
+	fmt.Println("")
+	return nil
+}
+
+// LaunchContainers launches a set of containers.
+func LaunchContainers(c incus.InstanceServer, count int, parallel int, image string, privileged bool, start bool, freeze bool) (time.Duration, error) {
+	var duration time.Duration
+
+	batchSize, err := getBatchSize(parallel)
+	if err != nil {
+		return duration, err
+	}
+
+	printTestConfig(count, batchSize, image, privileged, freeze)
+
+	fingerprint, err := ensureImage(c, image)
+	if err != nil {
+		return duration, err
+	}
+
+	batchStart := func(index int, wg *sync.WaitGroup) {
+		defer wg.Done()
+
+		name := getContainerName(count, index)
+
+		err := createContainer(c, fingerprint, name, privileged)
+		if err != nil {
+			logf("Failed to launch container '%s': %s", name, err)
+			return
+		}
+
+		if start {
+			err := startContainer(c, name)
+			if err != nil {
+				logf("Failed to start container '%s': %s", name, err)
+				return
+			}
+
+			if freeze {
+				err := freezeContainer(c, name)
+				if err != nil {
+					logf("Failed to freeze container '%s': %s", name, err)
+					return
+				}
+			}
+		}
+	}
+
+	duration = processBatch(count, batchSize, batchStart)
+	return duration, nil
+}
+
+// CreateContainers create the specified number of containers.
+func CreateContainers(c incus.InstanceServer, count int, parallel int, fingerprint string, privileged bool) (time.Duration, error) {
+	var duration time.Duration
+
+	batchSize, err := getBatchSize(parallel)
+	if err != nil {
+		return duration, err
+	}
+
+	batchCreate := func(index int, wg *sync.WaitGroup) {
+		defer wg.Done()
+
+		name := getContainerName(count, index)
+
+		err := createContainer(c, fingerprint, name, privileged)
+		if err != nil {
+			logf("Failed to launch container '%s': %s", name, err)
+			return
+		}
+	}
+
+	duration = processBatch(count, batchSize, batchCreate)
+
+	return duration, nil
+}
+
+// GetContainers returns containers created by the benchmark.
+func GetContainers(c incus.InstanceServer) ([]api.Instance, error) {
+	containers := []api.Instance{}
+
+	allContainers, err := c.GetInstances(api.InstanceTypeContainer)
+	if err != nil {
+		return containers, err
+	}
+
+	for _, container := range allContainers {
+		if container.Config[userConfigKey] == "true" {
+			containers = append(containers, container)
+		}
+	}
+
+	return containers, nil
+}
+
+// StartContainers starts containers created by the benchmark.
+func StartContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) {
+	var duration time.Duration
+
+	batchSize, err := getBatchSize(parallel)
+	if err != nil {
+		return duration, err
+	}
+
+	count := len(containers)
+	logf("Starting %d containers", count)
+
+	batchStart := func(index int, wg *sync.WaitGroup) {
+		defer wg.Done()
+
+		container := containers[index]
+		if !container.IsActive() {
+			err := startContainer(c, container.Name)
+			if err != nil {
+				logf("Failed to start container '%s': %s", container.Name, err)
+				return
+			}
+		}
+	}
+
+	duration = processBatch(count, batchSize, batchStart)
+	return duration, nil
+}
+
+// StopContainers stops containers created by the benchmark.
+func StopContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) {
+	var duration time.Duration
+
+	batchSize, err := getBatchSize(parallel)
+	if err != nil {
+		return duration, err
+	}
+
+	count := len(containers)
+	logf("Stopping %d containers", count)
+
+	batchStop := func(index int, wg *sync.WaitGroup) {
+		defer wg.Done()
+
+		container := containers[index]
+		if container.IsActive() {
+			err := stopContainer(c, container.Name)
+			if err != nil {
+				logf("Failed to stop container '%s': %s", container.Name, err)
+				return
+			}
+		}
+	}
+
+	duration = processBatch(count, batchSize, batchStop)
+	return duration, nil
+}
+
+// DeleteContainers removes containers created by the benchmark.
+func DeleteContainers(c incus.InstanceServer, containers []api.Instance, parallel int) (time.Duration, error) {
+	var duration time.Duration
+
+	batchSize, err := getBatchSize(parallel)
+	if err != nil {
+		return duration, err
+	}
+
+	count := len(containers)
+	logf("Deleting %d containers", count)
+
+	batchDelete := func(index int, wg *sync.WaitGroup) {
+		defer wg.Done()
+
+		container := containers[index]
+		name := container.Name
+		if container.IsActive() {
+			err := stopContainer(c, name)
+			if err != nil {
+				logf("Failed to stop container '%s': %s", name, err)
+				return
+			}
+		}
+
+		err = deleteContainer(c, name)
+		if err != nil {
+			logf("Failed to delete container: %s", name)
+			return
+		}
+	}
+
+	duration = processBatch(count, batchSize, batchDelete)
+	return duration, nil
+}
+
+func ensureImage(c incus.InstanceServer, image string) (string, error) {
+	var fingerprint string
+
+	if strings.Contains(image, ":") {
+		defaultConfig := config.NewConfig("", true)
+		defaultConfig.UserAgent = version.UserAgent
+
+		remote, fp, err := defaultConfig.ParseRemote(image)
+		if err != nil {
+			return "", err
+		}
+
+		fingerprint = fp
+
+		imageServer, err := defaultConfig.GetImageServer(remote)
+		if err != nil {
+			return "", err
+		}
+
+		if fingerprint == "" {
+			fingerprint = "default"
+		}
+
+		alias, _, err := imageServer.GetImageAlias(fingerprint)
+		if err == nil {
+			fingerprint = alias.Target
+		}
+
+		_, _, err = c.GetImage(fingerprint)
+		if err != nil {
+			logf("Importing image into local store: %s", fingerprint)
+			image, _, err := imageServer.GetImage(fingerprint)
+			if err != nil {
+				logf("Failed to import image: %s", err)
+				return "", err
+			}
+
+			err = copyImage(c, imageServer, *image)
+			if err != nil {
+				logf("Failed to import image: %s", err)
+				return "", err
+			}
+		}
+	} else {
+		fingerprint = image
+		alias, _, err := c.GetImageAlias(image)
+		if err == nil {
+			fingerprint = alias.Target
+		} else {
+			_, _, err = c.GetImage(image)
+		}
+
+		if err != nil {
+			logf("Image not found in local store: %s", image)
+			return "", err
+		}
+	}
+
+	logf("Found image in local store: %s", fingerprint)
+	return fingerprint, nil
+}

cmd/incus-benchmark/benchmark_batch.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"os"
+	"sync"
+	"time"
+)
+
+func getBatchSize(parallel int) (int, error) {
+	batchSize := parallel
+	if batchSize < 1 {
+		// Detect the number of parallel actions
+		cpus, err := os.ReadDir("/sys/bus/cpu/devices")
+		if err != nil {
+			return -1, err
+		}
+
+		batchSize = len(cpus)
+	}
+
+	return batchSize, nil
+}
+
+func processBatch(count int, batchSize int, process func(index int, wg *sync.WaitGroup)) time.Duration {
+	batches := count / batchSize
+	remainder := count % batchSize
+	processed := 0
+	wg := sync.WaitGroup{}
+	nextStat := batchSize
+
+	logf("Batch processing start")
+	timeStart := time.Now()
+
+	for i := 0; i < batches; i++ {
+		for j := 0; j < batchSize; j++ {
+			wg.Add(1)
+			go process(processed, &wg)
+			processed++
+		}
+
+		wg.Wait()
+
+		if processed >= nextStat {
+			interval := time.Since(timeStart).Seconds()
+			logf("Processed %d containers in %.3fs (%.3f/s)", processed, interval, float64(processed)/interval)
+			nextStat = nextStat * 2
+		}
+	}
+
+	for k := 0; k < remainder; k++ {
+		wg.Add(1)
+		go process(processed, &wg)
+		processed++
+	}
+
+	wg.Wait()
+
+	timeEnd := time.Now()
+	duration := timeEnd.Sub(timeStart)
+	logf("Batch processing completed in %.3fs", duration.Seconds())
+	return duration
+}

cmd/incus-benchmark/benchmark_operation.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"github.com/cyphar/incus/client"
+	"github.com/cyphar/incus/shared/api"
+)
+
+func createContainer(c incus.InstanceServer, fingerprint string, name string, privileged bool) error {
+	config := map[string]string{}
+	if privileged {
+		config["security.privileged"] = "true"
+	}
+
+	config[userConfigKey] = "true"
+
+	req := api.InstancesPost{
+		Name: name,
+		Source: api.InstanceSource{
+			Type:        "image",
+			Fingerprint: fingerprint,
+		},
+	}
+
+	req.Config = config
+
+	op, err := c.CreateInstance(req)
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}
+
+func startContainer(c incus.InstanceServer, name string) error {
+	op, err := c.UpdateInstanceState(
+		name, api.InstanceStatePut{Action: "start", Timeout: -1}, "")
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}
+
+func stopContainer(c incus.InstanceServer, name string) error {
+	op, err := c.UpdateInstanceState(
+		name, api.InstanceStatePut{Action: "stop", Timeout: -1, Force: true}, "")
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}
+
+func freezeContainer(c incus.InstanceServer, name string) error {
+	op, err := c.UpdateInstanceState(
+		name, api.InstanceStatePut{Action: "freeze", Timeout: -1}, "")
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}
+
+func deleteContainer(c incus.InstanceServer, name string) error {
+	op, err := c.DeleteInstance(name)
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}
+
+func copyImage(c incus.InstanceServer, s incus.ImageServer, image api.Image) error {
+	op, err := c.CopyImage(s, image, nil)
+	if err != nil {
+		return err
+	}
+
+	return op.Wait()
+}

cmd/incus-benchmark/benchmark_report.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+// Subset of JMeter CSV log format that are required by Jenkins performance
+// plugin
+// (see http://jmeter.apache.org/usermanual/listeners.html#csvlogformat)
+var csvFields = []string{
+	"timeStamp", // in milliseconds since 1/1/1970
+	"elapsed",   // in milliseconds
+	"label",
+	"responseCode",
+	"success", // "true" or "false"
+}
+
+// CSVReport reads/writes a CSV report file.
+type CSVReport struct {
+	Filename string
+
+	records [][]string
+}
+
+// Load reads current content of the filename and loads records.
+func (r *CSVReport) Load() error {
+	file, err := os.Open(r.Filename)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	reader := csv.NewReader(file)
+	for line := 1; err != io.EOF; line++ {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		err = r.addRecord(record)
+		if err != nil {
+			return err
+		}
+	}
+	logf("Loaded report file %s", r.Filename)
+	return nil
+}
+
+// Write writes current records to file.
+func (r *CSVReport) Write() error {
+	file, err := os.OpenFile(r.Filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = file.Close() }()
+
+	writer := csv.NewWriter(file)
+	err = writer.WriteAll(r.records)
+	if err != nil {
+		return err
+	}
+
+	logf("Written report file %s", r.Filename)
+	return file.Close()
+}
+
+// AddRecord adds a record to the report.
+func (r *CSVReport) AddRecord(label string, elapsed time.Duration) error {
+	if len(r.records) == 0 {
+		err := r.addRecord(csvFields)
+		if err != nil {
+			return err
+		}
+	}
+
+	record := []string{
+		fmt.Sprintf("%d", time.Now().UnixNano()/int64(time.Millisecond)), // timestamp
+		fmt.Sprintf("%d", elapsed/time.Millisecond),
+		label,
+		"",     // responseCode is not used
+		"true", // success"
+	}
+
+	return r.addRecord(record)
+}
+
+func (r *CSVReport) addRecord(record []string) error {
+	if len(record) != len(csvFields) {
+		return fmt.Errorf("Invalid number of fields : %q", record)
+	}
+
+	r.records = append(r.records, record)
+	return nil
+}

cmd/incus-benchmark/benchmark_util.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+func getContainerName(count int, index int) string {
+	nameFormat := "benchmark-%." + fmt.Sprintf("%d", len(fmt.Sprintf("%d", count))) + "d"
+	return fmt.Sprintf(nameFormat, index+1)
+}
+
+func logf(format string, args ...any) {
+	fmt.Printf(fmt.Sprintf("[%s] %s\n", time.Now().Format(time.StampMilli), format), args...)
+}
+
+func printTestConfig(count int, batchSize int, image string, privileged bool, freeze bool) {
+	privilegedStr := "unprivileged"
+	if privileged {
+		privilegedStr = "privileged"
+	}
+
+	mode := "normal startup"
+	if freeze {
+		mode = "start and freeze"
+	}
+
+	batches := count / batchSize
+	remainder := count % batchSize
+	fmt.Println("Test variables:")
+	fmt.Println("  Container count:", count)
+	fmt.Println("  Container mode:", privilegedStr)
+	fmt.Println("  Startup mode:", mode)
+	fmt.Println("  Image:", image)
+	fmt.Println("  Batches:", batches)
+	fmt.Println("  Batch size:", batchSize)
+	fmt.Println("  Remainder:", remainder)
+	fmt.Println("")
+}

cmd/incus-benchmark/main.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"os"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/cyphar/incus/client"
+	"github.com/cyphar/incus/shared"
+	"github.com/cyphar/incus/shared/version"
+)
+
+type cmdGlobal struct {
+	flagHelp        bool
+	flagParallel    int
+	flagProject     string
+	flagReportFile  string
+	flagReportLabel string
+	flagVersion     bool
+
+	srv            incus.InstanceServer
+	report         *CSVReport
+	reportDuration time.Duration
+}
+
+func (c *cmdGlobal) Run(cmd *cobra.Command, args []string) error {
+	// Connect to LXD
+	srv, err := incus.ConnectLXDUnix("", nil)
+	if err != nil {
+		return err
+	}
+
+	c.srv = srv.UseProject(c.flagProject)
+
+	// Print the initial header
+	err = PrintServerInfo(srv)
+	if err != nil {
+		return err
+	}
+
+	// Setup report handling
+	if c.flagReportFile != "" {
+		c.report = &CSVReport{Filename: c.flagReportFile}
+		if shared.PathExists(c.flagReportFile) {
+			err := c.report.Load()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *cmdGlobal) Teardown(cmd *cobra.Command, args []string) error {
+	// Nothing to do with not reporting
+	if c.report == nil {
+		return nil
+	}
+
+	label := cmd.Name()
+	if c.flagReportLabel != "" {
+		label = c.flagReportLabel
+	}
+
+	err := c.report.AddRecord(label, c.reportDuration)
+	if err != nil {
+		return err
+	}
+
+	err = c.report.Write()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	app := &cobra.Command{}
+	app.Use = "lxd-benchmark"
+	app.Short = "Benchmark performance of LXD"
+	app.Long = `Description:
+  Benchmark performance of LXD
+
+  This tool lets you benchmark various actions on a local LXD daemon.
+
+  It can be used just to check how fast a given LXD host is, to
+  compare performance on different servers or for performance tracking
+  when doing changes to the LXD codebase.
+
+  A CSV report can be produced to be consumed by graphing software.
+`
+	app.Example = `  # Spawn 20 containers in batches of 4
+  lxd-benchmark launch --count 20 --parallel 4
+
+  # Create 50 Alpine containers in batches of 10
+  lxd-benchmark init --count 50 --parallel 10 images:alpine/edge
+
+  # Delete all test containers using dynamic batch size
+  lxd-benchmark delete`
+	app.SilenceUsage = true
+	app.CompletionOptions = cobra.CompletionOptions{DisableDefaultCmd: true}
+
+	// Global flags
+	globalCmd := cmdGlobal{}
+	app.PersistentPreRunE = globalCmd.Run
+	app.PersistentPostRunE = globalCmd.Teardown
+	app.PersistentFlags().BoolVar(&globalCmd.flagVersion, "version", false, "Print version number")
+	app.PersistentFlags().BoolVarP(&globalCmd.flagHelp, "help", "h", false, "Print help")
+	app.PersistentFlags().IntVarP(&globalCmd.flagParallel, "parallel", "P", -1, "Number of threads to use"+"``")
+	app.PersistentFlags().StringVar(&globalCmd.flagReportFile, "report-file", "", "Path to the CSV report file"+"``")
+	app.PersistentFlags().StringVar(&globalCmd.flagReportLabel, "report-label", "", "Label for the new entry in the report [default=ACTION]"+"``")
+	app.PersistentFlags().StringVar(&globalCmd.flagProject, "project", "default", "Project to use")
+
+	// Version handling
+	app.SetVersionTemplate("{{.Version}}\n")
+	app.Version = version.Version
+
+	// init sub-command
+	initCmd := cmdInit{global: &globalCmd}
+	app.AddCommand(initCmd.Command())
+
+	// launch sub-command
+	launchCmd := cmdLaunch{global: &globalCmd, init: &initCmd}
+	app.AddCommand(launchCmd.Command())
+
+	// start sub-command
+	startCmd := cmdStart{global: &globalCmd}
+	app.AddCommand(startCmd.Command())
+
+	// stop sub-command
+	stopCmd := cmdStop{global: &globalCmd}
+	app.AddCommand(stopCmd.Command())
+
+	// delete sub-command
+	deleteCmd := cmdDelete{global: &globalCmd}
+	app.AddCommand(deleteCmd.Command())
+
+	// Run the main command and handle errors
+	err := app.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}

cmd/incus-benchmark/main_delete.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type cmdDelete struct {
+	global *cmdGlobal
+}
+
+func (c *cmdDelete) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "delete"
+	cmd.Short = "Delete containers"
+	cmd.RunE = c.Run
+
+	return cmd
+}
+
+func (c *cmdDelete) Run(cmd *cobra.Command, args []string) error {
+	// Get the containers
+	containers, err := GetContainers(c.global.srv)
+	if err != nil {
+		return err
+	}
+
+	// Run the test
+	duration, err := DeleteContainers(c.global.srv, containers, c.global.flagParallel)
+	if err != nil {
+		return err
+	}
+
+	c.global.reportDuration = duration
+
+	return nil
+}

cmd/incus-benchmark/main_init.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type cmdInit struct {
+	global *cmdGlobal
+
+	flagCount      int
+	flagPrivileged bool
+}
+
+func (c *cmdInit) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "init [[<remote>:]<image>]"
+	cmd.Short = "Create containers"
+	cmd.RunE = c.Run
+	cmd.Flags().IntVarP(&c.flagCount, "count", "C", 1, "Number of containers to create"+"``")
+	cmd.Flags().BoolVar(&c.flagPrivileged, "privileged", false, "Use privileged containers")
+
+	return cmd
+}
+
+func (c *cmdInit) Run(cmd *cobra.Command, args []string) error {
+	// Choose the image
+	image := "images:ubuntu/22.04"
+	if len(args) > 0 {
+		image = args[0]
+	}
+
+	// Run the test
+	duration, err := LaunchContainers(c.global.srv, c.flagCount, c.global.flagParallel, image, c.flagPrivileged, false, false)
+	if err != nil {
+		return err
+	}
+
+	c.global.reportDuration = duration
+
+	return nil
+}

cmd/incus-benchmark/main_launch.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type cmdLaunch struct {
+	global *cmdGlobal
+	init   *cmdInit
+
+	flagFreeze bool
+}
+
+func (c *cmdLaunch) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "launch [[<remote>:]<image>]"
+	cmd.Short = "Create and start containers"
+	cmd.RunE = c.Run
+	cmd.Flags().AddFlagSet(c.init.Command().Flags())
+	cmd.Flags().BoolVarP(&c.flagFreeze, "freeze", "F", false, "Freeze the container right after start")
+
+	return cmd
+}
+
+func (c *cmdLaunch) Run(cmd *cobra.Command, args []string) error {
+	// Choose the image
+	image := "images:ubuntu/22.04"
+	if len(args) > 0 {
+		image = args[0]
+	}
+
+	// Run the test
+	duration, err := LaunchContainers(c.global.srv, c.init.flagCount, c.global.flagParallel, image, c.init.flagPrivileged, true, c.flagFreeze)
+	if err != nil {
+		return err
+	}
+
+	c.global.reportDuration = duration
+
+	return nil
+}

cmd/incus-benchmark/main_start.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type cmdStart struct {
+	global *cmdGlobal
+}
+
+func (c *cmdStart) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "start"
+	cmd.Short = "Start containers"
+	cmd.RunE = c.Run
+
+	return cmd
+}
+
+func (c *cmdStart) Run(cmd *cobra.Command, args []string) error {
+	// Get the containers
+	containers, err := GetContainers(c.global.srv)
+	if err != nil {
+		return err
+	}
+
+	// Run the test
+	duration, err := StartContainers(c.global.srv, containers, c.global.flagParallel)
+	if err != nil {
+		return err
+	}
+
+	c.global.reportDuration = duration
+
+	return nil
+}

cmd/incus-benchmark/main_stop.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+type cmdStop struct {
+	global *cmdGlobal
+}
+
+func (c *cmdStop) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "stop"
+	cmd.Short = "Stop containers"
+	cmd.RunE = c.Run
+
+	return cmd
+}
+
+func (c *cmdStop) Run(cmd *cobra.Command, args []string) error {
+	// Get the containers
+	containers, err := GetContainers(c.global.srv)
+	if err != nil {
+		return err
+	}
+
+	// Run the test
+	duration, err := StopContainers(c.global.srv, containers, c.global.flagParallel)
+	if err != nil {
+		return err
+	}
+
+	c.global.reportDuration = duration
+
+	return nil
+}