Merge pull request #968 from arithx/qemu_2_electric_boogaloo

*: add unprivileged-qemu platform

README.md

@@ -295,3 +295,10 @@ for more information about the `.boto` file.
 
 ### qemu
 `qemu` is run locally and needs no credentials, but does need to be run as root.
+
+### qemu-unpriv
+`qemu-unpriv` is run locally and needs no credentials. It has a restricted set of functionality compared to the `qemu` platform, such as:
+
+- Single node only, no machine to machine networking
+- DHCP provides no data (forces several tests to be disabled)
+- No [Local cluster](platform/local/)

cmd/kola/options.go

@@ -30,7 +30,7 @@ var (
 	kolaPlatform       string
 	defaultTargetBoard = sdk.DefaultBoard()
 	kolaArchitectures  = []string{"amd64"}
-	kolaPlatforms      = []string{"aws", "do", "esx", "gce", "openstack", "packet", "qemu"}
+	kolaPlatforms      = []string{"aws", "do", "esx", "gce", "openstack", "packet", "qemu", "qemu-unpriv"}
 	kolaDistros        = []string{"cl", "fcos", "rhcos"}
 	kolaDefaultImages  = map[string]string{
 		"amd64-usr": sdk.BuildRoot() + "/images/amd64-usr/latest/coreos_production_image.bin",

cmd/kola/spawn.go

@@ -174,7 +174,7 @@ func doSpawn(cmd *cobra.Command, args []string) error {
 				return fmt.Errorf("Could not read machine options: %v", err)
 			}
 
-			var machineOpts qemu.MachineOptions
+			var machineOpts platform.MachineOptions
 			err = json.Unmarshal(b, &machineOpts)
 			if err != nil {
 				return fmt.Errorf("Could not unmarshal machine options: %v", err)

kola/harness.go

@@ -47,6 +47,7 @@ import (
 	"github.com/coreos/mantle/platform/machine/openstack"
 	"github.com/coreos/mantle/platform/machine/packet"
 	"github.com/coreos/mantle/platform/machine/qemu"
+	"github.com/coreos/mantle/platform/machine/unprivqemu"
 	"github.com/coreos/mantle/system"
 )
 
@@ -170,15 +171,24 @@ func NewFlight(pltfrm string) (flight platform.Flight, err error) {
 		flight, err = packet.NewFlight(&PacketOptions)
 	case "qemu":
 		flight, err = qemu.NewFlight(&QEMUOptions)
+	case "qemu-unpriv":
+		flight, err = unprivqemu.NewFlight(&QEMUOptions)
 	default:
 		err = fmt.Errorf("invalid platform %q", pltfrm)
 	}
 	return
 }
 
-func filterTests(tests map[string]*register.Test, pattern, platform string, version semver.Version) (map[string]*register.Test, error) {
+func filterTests(tests map[string]*register.Test, pattern, pltfrm string, version semver.Version) (map[string]*register.Test, error) {
 	r := make(map[string]*register.Test)
 
+	checkPlatforms := []string{pltfrm}
+
+	// qemu-unpriv has the same restrictions as QEMU but might also want additional restrictions due to the lack of a Local cluster
+	if pltfrm == "qemu-unpriv" {
+		checkPlatforms = append(checkPlatforms, "qemu")
+	}
+
 	for name, t := range tests {
 		match, err := filepath.Match(pattern, t.Name)
 		if err != nil {
@@ -202,13 +212,13 @@ func filterTests(tests map[string]*register.Test, pattern, platform string, vers
 			return false
 		}
 
-		if existsIn(platform, register.PlatformsNoInternet) && t.HasFlag(register.RequiresInternetAccess) {
-			plog.Infof("skipping test %s: Internet required but not supported by platform %s", t.Name, platform)
+		if existsIn(pltfrm, register.PlatformsNoInternet) && t.HasFlag(register.RequiresInternetAccess) {
+			plog.Debugf("skipping test %s: Internet required but not supported by platform %s", t.Name, pltfrm)
 			continue
 		}
 
-		isAllowed := func(item string, include, exclude []string) bool {
-			allowed := true
+		isAllowed := func(item string, include, exclude []string) (bool, bool) {
+			allowed, excluded := true, false
 			for _, i := range include {
 				if i == item {
 					allowed = true
@@ -220,20 +230,28 @@ func filterTests(tests map[string]*register.Test, pattern, platform string, vers
 			for _, i := range exclude {
 				if i == item {
 					allowed = false
+					excluded = true
 				}
 			}
-			return allowed
+			return allowed, excluded
 		}
 
-		if !isAllowed(platform, t.Platforms, t.ExcludePlatforms) {
+		isExcluded := false
+		allowed := false
+		for _, platform := range checkPlatforms {
+			allowedPlatform, excluded := isAllowed(platform, t.Platforms, t.ExcludePlatforms)
+			if excluded {
+				isExcluded = true
+				break
+			}
+			allowedArchitecture, _ := isAllowed(architecture(platform), t.Architectures, []string{})
+			allowed = allowed || (allowedPlatform && allowedArchitecture)
+		}
+		if isExcluded || !allowed {
 			continue
 		}
 
-		if !isAllowed(Options.Distribution, t.Distros, t.ExcludeDistros) {
-			continue
-		}
-
-		if !isAllowed(architecture(platform), t.Architectures, []string{}) {
+		if allowed, excluded := isAllowed(Options.Distribution, t.Distros, t.ExcludeDistros); !allowed || excluded {
 			continue
 		}
 

kola/register/register.go

@@ -37,6 +37,7 @@ var (
 	// platforms that have no Internet access
 	PlatformsNoInternet = []string{
 		"qemu",
+		"qemu-unpriv",
 	}
 )
 

kola/tests/docker/docker.go

@@ -59,6 +59,9 @@ func init() {
 		ClusterSize: 2,
 		Name:        "docker.network",
 		Distros:     []string{"cl"},
+
+		// qemu-unpriv machines cannot communicate
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 	register.Register(&register.Test{
 		Run:         dockerOldClient,
@@ -96,6 +99,9 @@ storage:
 passwd:
   users:
   - name: dockremap`),
+
+		// qemu-unpriv machines cannot communicate
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 
 	// This test covers all functionality that should be quick to run and can be
@@ -206,6 +212,12 @@ systemd:
   units:
    - name: docker.service
      enable: true`),
+
+		// https://github.com/coreos/mantle/issues/999
+		// On the qemu-unpriv platform the DHCP provides no data, pre-systemd 241 the DHCP server sending
+		// no routes to the link to spin in the configuring state. docker.service pulls in the network-online
+		// target which causes the basic machine checks to fail
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/docker/torcx_docker_flag.go

@@ -49,7 +49,8 @@ write_files:
   - path: "/etc/coreos/docker-1.12"
     content: yes
 `),
-		Distros: []string{"cl"},
+		Distros:          []string{"cl"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/ignition/resource.go

@@ -105,7 +105,7 @@ func init() {
 			"Serve": Serve,
 		},
 		// https://github.com/coreos/bugs/issues/2205
-		ExcludePlatforms: []string{"do"},
+		ExcludePlatforms: []string{"do", "qemu-unpriv"},
 		Distros:          []string{"cl", "fcos", "rhcos"},
 	})
 	register.Register(&register.Test{

kola/tests/ignition/systemd.go

@@ -65,6 +65,11 @@ func init() {
         }]
     }
 }`),
+		// https://github.com/coreos/mantle/issues/999
+		// On the qemu-unpriv platform the DHCP provides no data, pre-systemd 241 the DHCP server sending
+		// no routes to the link to spin in the configuring state. nfs-server.service pulls in the network-online
+		// target which causes the basic machine checks to fail
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/misc/cloudinit.go

@@ -32,7 +32,8 @@ hostname: "core1"
 write_files:
   - path: "/foo"
     content: bar`),
-		Distros: []string{"cl"},
+		Distros:          []string{"cl"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 	register.Register(&register.Test{
 		Run:         CloudInitScript,
@@ -47,7 +48,8 @@ EOF
 chown -R core.core ~core/.ssh
 chmod 700 ~core/.ssh
 chmod 600 ~core/.ssh/authorized_keys`),
-		Distros: []string{"cl"},
+		Distros:          []string{"cl"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/misc/grub.go

@@ -147,12 +147,13 @@ coreos:
 
 func init() {
 	register.Register(&register.Test{
-		Run:         UpdateGrubNop,
-		ClusterSize: 1,
-		Name:        "cl.update.grubnop",
-		UserData:    grubUpdaterConf,
-		MinVersion:  semver.Version{Major: 1745},
-		Distros:     []string{"cl"},
+		Run:              UpdateGrubNop,
+		ClusterSize:      1,
+		Name:             "cl.update.grubnop",
+		UserData:         grubUpdaterConf,
+		MinVersion:       semver.Version{Major: 1745},
+		Distros:          []string{"cl"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/misc/network.go

@@ -39,14 +39,16 @@ func init() {
   units:
     - name: docker.service
       enabled: true`),
-		MinVersion: semver.Version{Major: 1967},
+		MinVersion:       semver.Version{Major: 1967},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 	register.Register(&register.Test{
-		Run:         NetworkListeners,
-		ClusterSize: 1,
-		Name:        "cl.network.listeners.legacy",
-		Distros:     []string{"cl"},
-		EndVersion:  semver.Version{Major: 1967},
+		Run:              NetworkListeners,
+		ClusterSize:      1,
+		Name:             "cl.network.listeners.legacy",
+		Distros:          []string{"cl"},
+		EndVersion:       semver.Version{Major: 1967},
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 	register.Register(&register.Test{
 		Run:              NetworkInitramfsSecondBoot,

kola/tests/misc/nfs.go

@@ -53,6 +53,9 @@ func init() {
 		ClusterSize: 0,
 		Name:        "linux.nfs.v3",
 		Distros:     []string{"cl"},
+
+		// qemu-unpriv machines cannot communicate
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 	// TODO: enable FCOS when FCCT exists
 	register.Register(&register.Test{
@@ -60,6 +63,9 @@ func init() {
 		ClusterSize:    0,
 		Name:           "linux.nfs.v4",
 		ExcludeDistros: []string{"fcos"},
+
+		// qemu-unpriv machines cannot communicate
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/misc/ntp.go

@@ -26,11 +26,12 @@ import (
 
 func init() {
 	register.Register(&register.Test{
-		Run:         NTP,
-		ClusterSize: 0,
-		Name:        "linux.ntp",
-		Platforms:   []string{"qemu"},
-		Distros:     []string{"cl"},
+		Run:              NTP,
+		ClusterSize:      0,
+		Name:             "linux.ntp",
+		Platforms:        []string{"qemu"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
+		Distros:          []string{"cl"},
 	})
 }
 

kola/tests/misc/omaha.go

@@ -28,11 +28,12 @@ import (
 
 func init() {
 	register.Register(&register.Test{
-		Run:         OmahaPing,
-		ClusterSize: 0,
-		Name:        "cl.omaha.ping",
-		Platforms:   []string{"qemu"},
-		Distros:     []string{"cl"},
+		Run:              OmahaPing,
+		ClusterSize:      0,
+		Name:             "cl.omaha.ping",
+		Platforms:        []string{"qemu"},
+		ExcludePlatforms: []string{"qemu-unpriv"},
+		Distros:          []string{"cl"},
 	})
 }
 

kola/tests/misc/raid.go

@@ -22,6 +22,7 @@ import (
 	"github.com/coreos/mantle/platform"
 	"github.com/coreos/mantle/platform/conf"
 	"github.com/coreos/mantle/platform/machine/qemu"
+	"github.com/coreos/mantle/platform/machine/unprivqemu"
 )
 
 var (
@@ -105,12 +106,24 @@ systemd:
 }
 
 func RootOnRaid(c cluster.TestCluster) {
-	options := qemu.MachineOptions{
-		AdditionalDisks: []qemu.Disk{
+	var m platform.Machine
+	var err error
+	options := platform.MachineOptions{
+		AdditionalDisks: []platform.Disk{
 			{Size: "520M", DeviceOpts: []string{"serial=secondary"}},
 		},
 	}
-	m, err := c.Cluster.(*qemu.Cluster).NewMachineWithOptions(raidRootUserData, options)
+	switch pc := c.Cluster.(type) {
+	// These cases have to be separated because when put together to the same case statement
+	// the golang compiler no longer checks that the individual types in the case have the
+	// NewMachineWithOptions function, but rather whether platform.Cluster does which fails
+	case *qemu.Cluster:
+		m, err = pc.NewMachineWithOptions(raidRootUserData, options)
+	case *unprivqemu.Cluster:
+		m, err = pc.NewMachineWithOptions(raidRootUserData, options)
+	default:
+		c.Fatal("unknown cluster type")
+	}
 	if err != nil {
 		c.Fatal(err)
 	}

kola/tests/systemd/journald.go

@@ -59,6 +59,9 @@ func init() {
 		ClusterSize: 0,
 		Name:        "systemd.journal.remote",
 		Distros:     []string{"cl"},
+
+		// qemu-unpriv machines cannot communicate
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

kola/tests/torcx/torcx.go

@@ -35,6 +35,12 @@ systemd:
     enable: true
 `),
 		Distros: []string{"cl"},
+
+		// https://github.com/coreos/mantle/issues/999
+		// On the qemu-unpriv platform the DHCP provides no data, pre-systemd 241 the DHCP server sending
+		// no routes to the link to spin in the configuring state. docker.service pulls in the network-online
+		// target which causes the basic machine checks to fail
+		ExcludePlatforms: []string{"qemu-unpriv"},
 	})
 }
 

platform/machine/qemu/cluster.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"sync"
 
@@ -41,15 +40,11 @@ type Cluster struct {
 	*local.LocalCluster
 }
 
-type MachineOptions struct {
-	AdditionalDisks []Disk
-}
-
 func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) {
-	return qc.NewMachineWithOptions(userdata, MachineOptions{})
+	return qc.NewMachineWithOptions(userdata, platform.MachineOptions{})
 }
 
-func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options MachineOptions) (platform.Machine, error) {
+func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) {
 	id := uuid.New()
 
 	dir := filepath.Join(qc.RuntimeConf().OutputDir, id)
@@ -99,86 +94,15 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options Machin
 		consolePath: filepath.Join(dir, "console.txt"),
 	}
 
-	var qmCmd []string
-	combo := runtime.GOARCH + "--" + qc.flight.opts.Board
-	switch combo {
-	case "amd64--amd64-usr":
-		qmCmd = []string{
-			"qemu-system-x86_64",
-			"-machine", "accel=kvm",
-			"-cpu", "host",
-			"-m", "1024",
-		}
-	case "amd64--arm64-usr":
-		qmCmd = []string{
-			"qemu-system-aarch64",
-			"-machine", "virt",
-			"-cpu", "cortex-a57",
-			"-m", "2048",
-		}
-	case "arm64--amd64-usr":
-		qmCmd = []string{
-			"qemu-system-x86_64",
-			"-machine", "pc-q35-2.8",
-			"-cpu", "kvm64",
-			"-m", "1024",
-		}
-	case "arm64--arm64-usr":
-		qmCmd = []string{
-			"qemu-system-aarch64",
-			"-machine", "virt,accel=kvm,gic-version=3",
-			"-cpu", "host",
-			"-m", "2048",
-		}
-	default:
-		panic("host-guest combo not supported: " + combo)
+	qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, qc.flight.opts.BIOSImage, qm.consolePath, confPath, qc.flight.diskImagePath, conf.IsIgnition(), options)
+	if err != nil {
+		return nil, err
 	}
 
+	for _, file := range extraFiles {
+		defer file.Close()
+	}
 	qmMac := qm.netif.HardwareAddr.String()
-	qmCmd = append(qmCmd,
-		"-bios", qc.flight.opts.BIOSImage,
-		"-smp", "1",
-		"-uuid", qm.id,
-		"-display", "none",
-		"-chardev", "file,id=log,path="+qm.consolePath,
-		"-serial", "chardev:log",
-	)
-
-	if conf.IsIgnition() {
-		qmCmd = append(qmCmd,
-			"-fw_cfg", "name=opt/com.coreos/config,file="+confPath)
-	} else {
-		qmCmd = append(qmCmd,
-			"-fsdev", "local,id=cfg,security_model=none,readonly,path="+confPath,
-			"-device", qc.virtio("9p", "fsdev=cfg,mount_tag=config-2"))
-	}
-
-	allDisks := append([]Disk{
-		{
-			BackingFile: qc.flight.diskImagePath,
-			DeviceOpts:  primaryDiskOptions,
-		},
-	}, options.AdditionalDisks...)
-
-	var extraFiles []*os.File
-	fdnum := 3 // first additional file starts at position 3
-	fdset := 1
-
-	for _, disk := range allDisks {
-		optionsDiskFile, err := disk.setupFile()
-		if err != nil {
-			return nil, err
-		}
-		defer optionsDiskFile.Close()
-		extraFiles = append(extraFiles, optionsDiskFile)
-
-		id := fmt.Sprintf("d%d", fdnum)
-		qmCmd = append(qmCmd, "-add-fd", fmt.Sprintf("fd=%d,set=%d", fdnum, fdset),
-			"-drive", fmt.Sprintf("if=none,id=%s,format=qcow2,file=/dev/fdset/%d", id, fdset),
-			"-device", qc.virtio("blk", fmt.Sprintf("drive=%s%s", id, disk.getOpts())))
-		fdnum += 1
-		fdset += 1
-	}
 
 	qc.mu.Lock()
 
@@ -188,12 +112,13 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options Machin
 		return nil, err
 	}
 	defer tap.Close()
+	fdnum := 3 + len(extraFiles)
 	qmCmd = append(qmCmd, "-netdev", fmt.Sprintf("tap,id=tap,fd=%d", fdnum),
-		"-device", qc.virtio("net", "netdev=tap,mac="+qmMac))
+		"-device", platform.Virtio(qc.flight.opts.Board, "net", "netdev=tap,mac="+qmMac))
 	fdnum += 1
 	extraFiles = append(extraFiles, tap.File)
 
-	plog.Debugf("NewMachine: (%s) %q", combo, qmCmd)
+	plog.Debugf("NewMachine: %q", qmCmd)
 
 	qm.qemu = qm.qc.NewCommand(qmCmd[0], qmCmd[1:]...)
 
@@ -218,21 +143,6 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options Machin
 	return qm, nil
 }
 
-// The virtio device name differs between machine types but otherwise
-// configuration is the same. Use this to help construct device args.
-func (qc *Cluster) virtio(device, args string) string {
-	var suffix string
-	switch qc.flight.opts.Board {
-	case "amd64-usr":
-		suffix = "pci"
-	case "arm64-usr":
-		suffix = "device"
-	default:
-		panic(qc.flight.opts.Board)
-	}
-	return fmt.Sprintf("virtio-%s-%s,%s", device, suffix, args)
-}
-
 func (qc *Cluster) Destroy() {
 	qc.LocalCluster.Destroy()
 	qc.flight.DelCluster(qc)

platform/machine/qemu/flight.go

@@ -81,7 +81,7 @@ func NewFlight(opts *Options) (platform.Flight, error) {
 			return nil, err
 		}
 		if info.Format != "raw" {
-			// makeCLDiskTemplate() needs to be able to mount
+			// platform.MakeCLDiskTemplate() needs to be able to mount
 			// partitions
 			plog.Debug("disk image is in qcow format; not enabling console logging")
 			opts.UseVanillaImage = true
@@ -89,7 +89,7 @@ func NewFlight(opts *Options) (platform.Flight, error) {
 	}
 	if !opts.UseVanillaImage {
 		plog.Debug("enabling console logging in base disk")
-		qf.diskImageFile, err = makeCLDiskTemplate(opts.DiskImage)
+		qf.diskImageFile, err = platform.MakeCLDiskTemplate(opts.DiskImage)
 		if err != nil {
 			qf.Destroy()
 			return nil, err

platform/machine/unprivqemu/cluster.go

@@ -0,0 +1,198 @@
+// Copyright 2019 Red Hat
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package unprivqemu
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pborman/uuid"
+
+	"github.com/coreos/mantle/platform"
+	"github.com/coreos/mantle/platform/conf"
+	"github.com/coreos/mantle/system/exec"
+	"github.com/coreos/mantle/util"
+)
+
+// Cluster is a local cluster of QEMU-based virtual machines.
+//
+// XXX: must be exported so that certain QEMU tests can access struct members
+// through type assertions.
+type Cluster struct {
+	*platform.BaseCluster
+	flight *flight
+
+	mu sync.Mutex
+}
+
+func (qc *Cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) {
+	return qc.NewMachineWithOptions(userdata, platform.MachineOptions{})
+}
+
+func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) {
+	id := uuid.New()
+
+	dir := filepath.Join(qc.RuntimeConf().OutputDir, id)
+	if err := os.Mkdir(dir, 0777); err != nil {
+		return nil, err
+	}
+
+	// hacky solution for cloud config ip substitution
+	// NOTE: escaping is not supported
+	qc.mu.Lock()
+
+	conf, err := qc.RenderUserData(userdata, map[string]string{})
+	if err != nil {
+		qc.mu.Unlock()
+		return nil, err
+	}
+	qc.mu.Unlock()
+
+	var confPath string
+	if conf.IsIgnition() {
+		confPath = filepath.Join(dir, "ignition.json")
+		if err := conf.WriteFile(confPath); err != nil {
+			return nil, err
+		}
+	} else if conf.IsEmpty() {
+	} else {
+		return nil, fmt.Errorf("unprivileged qemu only supports Ignition or empty configs")
+	}
+
+	journal, err := platform.NewJournal(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	qm := &machine{
+		qc:          qc,
+		id:          id,
+		journal:     journal,
+		consolePath: filepath.Join(dir, "console.txt"),
+	}
+
+	qmCmd, extraFiles, err := platform.CreateQEMUCommand(qc.flight.opts.Board, qm.id, qc.flight.opts.BIOSImage, qm.consolePath, confPath, qc.flight.diskImagePath, conf.IsIgnition(), options)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range extraFiles {
+		defer file.Close()
+	}
+
+	qc.mu.Lock()
+
+	qmCmd = append(qmCmd, "-netdev", "user,id=eth0,restrict=yes,hostfwd=tcp:127.0.0.1:0-:22", "-device", platform.Virtio(qc.flight.opts.Board, "net", "netdev=eth0"))
+
+	plog.Debugf("NewMachine: %q", qmCmd)
+
+	qm.qemu = exec.Command(qmCmd[0], qmCmd[1:]...)
+
+	qc.mu.Unlock()
+
+	cmd := qm.qemu.(*exec.ExecCmd)
+	cmd.Stderr = os.Stderr
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, extraFiles...)
+
+	if err = qm.qemu.Start(); err != nil {
+		return nil, err
+	}
+
+	pid := strconv.Itoa(qm.qemu.Pid())
+	err = util.Retry(6, 5*time.Second, func() error {
+		var err error
+		qm.ip, err = getAddress(pid)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := platform.StartMachine(qm, qm.journal); err != nil {
+		qm.Destroy()
+		return nil, err
+	}
+
+	qc.AddMach(qm)
+
+	return qm, nil
+}
+
+func (qc *Cluster) Destroy() {
+	qc.BaseCluster.Destroy()
+	qc.flight.DelCluster(qc)
+}
+
+// parse /proc/net/tcp to determine the port selected by QEMU
+func getAddress(pid string) (string, error) {
+	data, err := ioutil.ReadFile("/proc/net/tcp")
+	if err != nil {
+		return "", fmt.Errorf("reading /proc/net/tcp: %v", err)
+	}
+
+	for _, line := range strings.Split(string(data), "\n")[1:] {
+		fields := strings.Fields(line)
+		if len(fields) < 10 {
+			// at least 10 fields are neeeded for the local & remote address and the inode
+			continue
+		}
+		localAddress := fields[1]
+		remoteAddress := fields[2]
+		inode := fields[9]
+
+		isLocalPat := regexp.MustCompile("0100007F:[[:xdigit:]]{4}")
+		if !isLocalPat.MatchString(localAddress) || remoteAddress != "00000000:0000" {
+			continue
+		}
+
+		dir := fmt.Sprintf("/proc/%s/fd/", pid)
+		fds, err := ioutil.ReadDir(dir)
+		if err != nil {
+			return "", fmt.Errorf("listing %s: %v", dir, err)
+		}
+
+		for _, f := range fds {
+			link, err := os.Readlink(filepath.Join(dir, f.Name()))
+			if err != nil {
+				continue
+			}
+			socketPattern := regexp.MustCompile("socket:\\[([0-9]+)\\]")
+			match := socketPattern.FindStringSubmatch(link)
+			if len(match) > 1 {
+				if inode == match[1] {
+					// this entry belongs to the QEMU pid, parse the port and return the address
+					portHex := strings.Split(localAddress, ":")[1]
+					port, err := strconv.ParseInt(portHex, 16, 32)
+					if err != nil {
+						return "", fmt.Errorf("decoding port %q: %v", portHex, err)
+					}
+					return fmt.Sprintf("127.0.0.1:%d", port), nil
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("didn't find an address")
+}

platform/machine/unprivqemu/flight.go

@@ -0,0 +1,79 @@
+// Copyright 2019 Red Hat
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package unprivqemu
+
+import (
+	"os"
+
+	"github.com/coreos/pkg/capnslog"
+
+	"github.com/coreos/mantle/platform"
+	"github.com/coreos/mantle/platform/machine/qemu"
+)
+
+const (
+	Platform platform.Name = "qemu"
+)
+
+type flight struct {
+	*platform.BaseFlight
+	opts *qemu.Options
+
+	diskImagePath string
+	diskImageFile *os.File
+}
+
+var (
+	plog = capnslog.NewPackageLogger("github.com/coreos/mantle", "platform/machine/qemu")
+)
+
+func NewFlight(opts *qemu.Options) (platform.Flight, error) {
+	bf, err := platform.NewBaseFlight(opts.Options, Platform, "")
+	if err != nil {
+		return nil, err
+	}
+
+	qf := &flight{
+		BaseFlight:    bf,
+		opts:          opts,
+		diskImagePath: opts.DiskImage,
+	}
+
+	return qf, nil
+}
+
+// NewCluster creates a Cluster instance, suitable for running virtual
+// machines in QEMU.
+func (qf *flight) NewCluster(rconf *platform.RuntimeConfig) (platform.Cluster, error) {
+	bc, err := platform.NewBaseCluster(qf.BaseFlight, rconf)
+	if err != nil {
+		return nil, err
+	}
+
+	qc := &Cluster{
+		BaseCluster: bc,
+		flight:      qf,
+	}
+
+	qf.AddCluster(qc)
+
+	return qc, nil
+}
+
+func (qf *flight) Destroy() {
+	if qf.diskImageFile != nil {
+		qf.diskImageFile.Close()
+	}
+}

platform/machine/unprivqemu/machine.go

@@ -0,0 +1,98 @@
+// Copyright 2019 Red Hat
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package unprivqemu
+
+import (
+	"io/ioutil"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/coreos/mantle/platform"
+	"github.com/coreos/mantle/system/exec"
+)
+
+type machine struct {
+	qc          *Cluster
+	id          string
+	qemu        exec.Cmd
+	journal     *platform.Journal
+	consolePath string
+	console     string
+	ip          string
+}
+
+func (m *machine) ID() string {
+	return m.id
+}
+
+func (m *machine) IP() string {
+	return m.ip
+}
+
+func (m *machine) PrivateIP() string {
+	return m.ip
+}
+
+func (m *machine) RuntimeConf() platform.RuntimeConfig {
+	return m.qc.RuntimeConf()
+}
+
+func (m *machine) SSHClient() (*ssh.Client, error) {
+	return m.qc.SSHClient(m.IP())
+}
+
+func (m *machine) PasswordSSHClient(user string, password string) (*ssh.Client, error) {
+	return m.qc.PasswordSSHClient(m.IP(), user, password)
+}
+
+func (m *machine) SSH(cmd string) ([]byte, []byte, error) {
+	return m.qc.SSH(m, cmd)
+}
+
+func (m *machine) Reboot() error {
+	return platform.RebootMachine(m, m.journal)
+}
+
+func (m *machine) Destroy() {
+	if err := m.qemu.Kill(); err != nil {
+		plog.Errorf("Error killing instance %v: %v", m.ID(), err)
+	}
+
+	m.journal.Destroy()
+
+	if buf, err := ioutil.ReadFile(m.consolePath); err == nil {
+		m.console = string(buf)
+	} else {
+		plog.Errorf("Error reading console for instance %v: %v", m.ID(), err)
+	}
+
+	m.qc.DelMach(m)
+}
+
+func (m *machine) ConsoleOutput() string {
+	return m.console
+}
+
+func (m *machine) JournalOutput() string {
+	if m.journal == nil {
+		return ""
+	}
+
+	data, err := m.journal.Read()
+	if err != nil {
+		plog.Errorf("Reading journal for instance %v: %v", m.ID(), err)
+	}
+	return string(data)
+}

platform/qemu.go

@@ -1,4 +1,4 @@
-// Copyright 2017 CoreOS, Inc.
+// Copyright 2019 Red Hat
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package qemu
+package platform
 
 import (
 	"errors"
@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -27,6 +28,10 @@ import (
 	"github.com/coreos/mantle/util"
 )
 
+type MachineOptions struct {
+	AdditionalDisks []Disk
+}
+
 type Disk struct {
 	Size        string   // disk image size in bytes, optional suffixes "K", "M", "G", "T" allowed. Incompatible with BackingFile
 	BackingFile string   // raw disk image to use. Incompatible with Size.
@@ -42,7 +47,7 @@ var (
 // Copy Container Linux input image and specialize copy for running kola tests.
 // Return FD to the copy, which is a deleted file.
 // This is not mandatory; the tests will do their best without it.
-func makeCLDiskTemplate(inputPath string) (output *os.File, result error) {
+func MakeCLDiskTemplate(inputPath string) (output *os.File, result error) {
 	seterr := func(err error) {
 		if result == nil {
 			result = err
@@ -214,3 +219,107 @@ func mkpath(basedir string) (string, error) {
 	defer f.Close()
 	return f.Name(), nil
 }
+
+func CreateQEMUCommand(board, uuid, biosImage, consolePath, confPath, diskImagePath string, isIgnition bool, options MachineOptions) ([]string, []*os.File, error) {
+	var qmCmd []string
+
+	// As we expand this list of supported native + board
+	// archs combos we should coordinate with the
+	// coreos-assembler folks as they utilize something
+	// similar in cosa run
+	combo := runtime.GOARCH + "--" + board
+	switch combo {
+	case "amd64--amd64-usr":
+		qmCmd = []string{
+			"qemu-system-x86_64",
+			"-machine", "accel=kvm",
+			"-cpu", "host",
+			"-m", "1024",
+		}
+	case "amd64--arm64-usr":
+		qmCmd = []string{
+			"qemu-system-aarch64",
+			"-machine", "virt",
+			"-cpu", "cortex-a57",
+			"-m", "2048",
+		}
+	case "arm64--amd64-usr":
+		qmCmd = []string{
+			"qemu-system-x86_64",
+			"-machine", "pc-q35-2.8",
+			"-cpu", "kvm64",
+			"-m", "1024",
+		}
+	case "arm64--arm64-usr":
+		qmCmd = []string{
+			"qemu-system-aarch64",
+			"-machine", "virt,accel=kvm,gic-version=3",
+			"-cpu", "host",
+			"-m", "2048",
+		}
+	default:
+		panic("host-guest combo not supported: " + combo)
+	}
+
+	qmCmd = append(qmCmd,
+		"-bios", biosImage,
+		"-smp", "1",
+		"-uuid", uuid,
+		"-display", "none",
+		"-chardev", "file,id=log,path="+consolePath,
+		"-serial", "chardev:log",
+	)
+
+	if isIgnition {
+		qmCmd = append(qmCmd,
+			"-fw_cfg", "name=opt/com.coreos/config,file="+confPath)
+	} else {
+		qmCmd = append(qmCmd,
+			"-fsdev", "local,id=cfg,security_model=none,readonly,path="+confPath,
+			"-device", Virtio(board, "9p", "fsdev=cfg,mount_tag=config-2"))
+	}
+
+	allDisks := append([]Disk{
+		{
+			BackingFile: diskImagePath,
+			DeviceOpts:  primaryDiskOptions,
+		},
+	}, options.AdditionalDisks...)
+
+	var extraFiles []*os.File
+	fdnum := 3 // first additional file starts at position 3
+	fdset := 1
+
+	for _, disk := range allDisks {
+		optionsDiskFile, err := disk.setupFile()
+		if err != nil {
+			return nil, nil, err
+		}
+		//defer optionsDiskFile.Close()
+		extraFiles = append(extraFiles, optionsDiskFile)
+
+		id := fmt.Sprintf("d%d", fdnum)
+		qmCmd = append(qmCmd, "-add-fd", fmt.Sprintf("fd=%d,set=%d", fdnum, fdset),
+			"-drive", fmt.Sprintf("if=none,id=%s,format=qcow2,file=/dev/fdset/%d", id, fdset),
+			"-device", Virtio(board, "blk", fmt.Sprintf("drive=%s%s", id, disk.getOpts())))
+		fdnum += 1
+		fdset += 1
+	}
+
+	return qmCmd, extraFiles, nil
+}
+
+// The virtio device name differs between machine types but otherwise
+// configuration is the same. Use this to help construct device args.
+func Virtio(board, device, args string) string {
+	var suffix string
+	switch board {
+	case "amd64-usr":
+		suffix = "pci"
+	case "arm64-usr":
+		suffix = "device"
+	default:
+		panic(board)
+	}
+	return fmt.Sprintf("virtio-%s-%s,%s", device, suffix, args)
+}

system/exec/exec.go

@@ -43,6 +43,9 @@ type Cmd interface {
 
 	// Simplified wrapper for Process.Kill + Wait
 	Kill() error
+
+	// Simplified wrapper for Process.Pid
+	Pid() int
 }
 
 // Basic Cmd implementation based on exec.Cmd
@@ -80,6 +83,10 @@ func (cmd *ExecCmd) Kill() error {
 	return err
 }
 
+func (cmd *ExecCmd) Pid() int {
+	return cmd.Process.Pid
+}
+
 // IsCmdNotFound reports true if the underlying error was exec.ErrNotFound.
 func IsCmdNotFound(err error) bool {
 	if eerr, ok := err.(*exec.Error); ok && eerr.Err == ErrNotFound {