lxc/incus
1
0
Fork 0
mirror of https://github.com/lxc/incus.git synced 2026-02-05 09:46:19 +01:00
Code Issues Releases Activity

incus-agent: Split OS specific logic

Browse Source 
This moves all OS specific logic to a new os_OSNAME.go file.

It also introduces a basic placeholder for Windows, allowing the agent
to be built on Windows, even if most features will return "Not implemented"
at this stage.

Signed-off-by: Stéphane Graber <stgraber@stgraber.org>
This commit is contained in:
Stéphane Graber
2025-04-16 01:16:57 -04:00
parent 7445702d7c
commit 6e22aa905f
8 changed files with 996 additions and 941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cmd/incus-agent/api_1.0.go
View File

@@ -8,17 +8,13 @@ import (
"net/http"
"os"
"github.com/mdlayher/vsock"
incus "github.com/lxc/incus/v6/client"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/ports"
"github.com/lxc/incus/v6/internal/server/response"
localvsock "github.com/lxc/incus/v6/internal/server/vsock"
"github.com/lxc/incus/v6/internal/version"
"github.com/lxc/incus/v6/shared/api"
agentAPI "github.com/lxc/incus/v6/shared/api/agent"
"github.com/lxc/incus/v6/shared/logger"
localtls "github.com/lxc/incus/v6/shared/tls"
)
@@ -50,28 +46,13 @@ func api10Get(d *Daemon, r *http.Request) response.Response {
AuthMethods: []string{api.AuthenticationMethodTLS},
}
uname, err := linux.Uname()
env, err := osGetEnvironment()
if err != nil {
return response.InternalError(err)
}
serverName, err := os.Hostname()
if err != nil {
return response.SmartError(err)
}
env := api.ServerEnvironment{
Kernel: uname.Sysname,
KernelArchitecture: uname.Machine,
KernelVersion: uname.Release,
Server: "incus-agent",
ServerPid: os.Getpid(),
ServerVersion: version.Version,
ServerName: serverName,
}
fullSrv := api.Server{ServerUntrusted: srv}
fullSrv.Environment = env
fullSrv.Environment = *env
return response.SyncResponseETag(true, fullSrv, fullSrv)
}
@@ -198,19 +179,11 @@ func getClient(CID uint32, port int, serverCertificate string) (*http.Client, er
}
func startHTTPServer(d *Daemon, debug bool) error {
const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY.
// Setup the listener on wildcard CID for inbound connections from Incus.
// We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works.
// A CID change can occur when restoring a stateful VM that was previously using one CID but is
// subsequently restored using a different one.
l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil)
l, err := osGetListener(ports.HTTPSDefaultPort)
if err != nil {
return fmt.Errorf("Failed to listen on vsock: %w", err)
return fmt.Errorf("Failed to get listener: %w", err)
}
logger.Info("Started vsock listener")
// Load the expected server certificate.
cert, err := localtls.ReadCert("server.crt")
if err != nil {

2
cmd/incus-agent/events.go
View File

@@ -151,7 +151,7 @@ func eventsProcess(event api.Event) {
for i := 0; i < 20; i++ {
time.Sleep(500 * time.Millisecond)
err = tryMountShared(mntSource, e.Config["path"], "virtiofs", nil)
err = osMountShared(mntSource, e.Config["path"], "virtiofs", nil)
if err == nil {
break
}

122
cmd/incus-agent/exec.go
View File

@@ -12,21 +12,17 @@ import (
"os/exec"
"strconv"
"sync"
"syscall"
"time"
"github.com/gorilla/websocket"
"golang.org/x/sys/unix"
"github.com/lxc/incus/v6/internal/jmap"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/server/db/operationtype"
"github.com/lxc/incus/v6/internal/server/operations"
"github.com/lxc/incus/v6/internal/server/response"
internalUtil "github.com/lxc/incus/v6/internal/util"
"github.com/lxc/incus/v6/shared/api"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/util"
"github.com/lxc/incus/v6/shared/ws"
)
@@ -69,44 +65,7 @@ func execPost(d *Daemon, r *http.Request) response.Response {
}
}
// Set default value for PATH
_, ok := env["PATH"]
if !ok {
env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
}
if util.PathExists("/snap/bin") {
env["PATH"] = fmt.Sprintf("%s:/snap/bin", env["PATH"])
}
// If running as root, set some env variables
if post.User == 0 {
// Set default value for HOME
_, ok = env["HOME"]
if !ok {
env["HOME"] = "/root"
}
// Set default value for USER
_, ok = env["USER"]
if !ok {
env["USER"] = "root"
}
}
// Set default value for LANG
_, ok = env["LANG"]
if !ok {
env["LANG"] = "C.UTF-8"
}
// Set the default working directory
if post.Cwd == "" {
post.Cwd = env["HOME"]
if post.Cwd == "" {
post.Cwd = "/"
}
}
osSetEnv(&post, env)
ws := &execWs{}
ws.fds = map[int]string{}
@@ -260,7 +219,8 @@ func (s *execWs) Do(op *operations.Operation) error {
if s.interactive {
ttys = make([]*os.File, 1)
ptys = make([]*os.File, 1)
ptys[0], ttys[0], err = linux.OpenPty(int64(s.uid), int64(s.gid))
ptys[0], ttys[0], err = osGetInteractiveConsole(s)
if err != nil {
return err
}
@@ -268,10 +228,6 @@ func (s *execWs) Do(op *operations.Operation) error {
stdin = ttys[0]
stdout = ttys[0]
stderr = ttys[0]
if s.width > 0 && s.height > 0 {
_ = linux.SetPtySize(int(ptys[0].Fd()), s.width, s.height)
}
} else {
ttys = make([]*os.File, 3)
ptys = make([]*os.File, 3)
@@ -287,10 +243,14 @@ func (s *execWs) Do(op *operations.Operation) error {
stderr = ttys[execWSStderr]
}
ctxCommand, cancel := context.WithCancel(context.Background())
waitAttachedChildIsDead, markAttachedChildIsDead := context.WithCancel(context.Background())
var wgEOF sync.WaitGroup
finisher := func(cmdResult int, cmdErr error) error {
// Cancel the context after we're done with cleanup.
defer cancel()
// Cancel this before closing the control connection so control handler can detect command ending.
markAttachedChildIsDead()
@@ -324,9 +284,9 @@ func (s *execWs) Do(op *operations.Operation) error {
var cmd *exec.Cmd
if len(s.command) > 1 {
cmd = exec.Command(s.command[0], s.command[1:]...)
cmd = exec.CommandContext(ctxCommand, s.command[0], s.command[1:]...)
} else {
cmd = exec.Command(s.command[0])
cmd = exec.CommandContext(ctxCommand, s.command[0])
}
// Prepare the environment
@@ -337,27 +297,10 @@ func (s *execWs) Do(op *operations.Operation) error {
cmd.Stdin = stdin
cmd.Stdout = stdout
cmd.Stderr = stderr
cmd.SysProcAttr = &syscall.SysProcAttr{
Credential: &syscall.Credential{
Uid: s.uid,
Gid: s.gid,
},
// Creates a new session if the calling process is not a process group leader.
// The calling process is the leader of the new session, the process group leader of
// the new process group, and has no controlling terminal.
// This is important to allow remote shells to handle ctrl+c.
Setsid: true,
}
// Make the given terminal the controlling terminal of the calling process.
// The calling process must be a session leader and not have a controlling terminal already.
// This is important as allows ctrl+c to work as expected for non-shell programs.
if s.interactive {
cmd.SysProcAttr.Setctty = true
}
cmd.Dir = s.cwd
osPrepareExecCommand(s, cmd)
err = cmd.Start()
if err != nil {
exitStatus := -1
@@ -399,12 +342,7 @@ func (s *execWs) Do(op *operations.Operation) error {
l.Warn("Failed getting exec control websocket reader, killing command", logger.Ctx{"err": err})
}
err := unix.Kill(cmd.Process.Pid, unix.SIGKILL)
if err != nil {
l.Error("Failed to send SIGKILL")
} else {
l.Info("Sent SIGKILL")
}
cancel()
return
}
@@ -421,40 +359,14 @@ func (s *execWs) Do(op *operations.Operation) error {
return
}
command := api.InstanceExecControl{}
err = json.Unmarshal(buf, &command)
control := api.InstanceExecControl{}
err = json.Unmarshal(buf, &control)
if err != nil {
l.Debug("Failed to unmarshal control socket command", logger.Ctx{"err": err})
continue
}
if command.Command == "window-resize" && s.interactive {
winchWidth, err := strconv.Atoi(command.Args["width"])
if err != nil {
l.Debug("Unable to extract window width", logger.Ctx{"err": err})
continue
}
winchHeight, err := strconv.Atoi(command.Args["height"])
if err != nil {
l.Debug("Unable to extract window height", logger.Ctx{"err": err})
continue
}
err = linux.SetPtySize(int(ptys[0].Fd()), winchWidth, winchHeight)
if err != nil {
l.Debug("Failed to set window size", logger.Ctx{"err": err, "width": winchWidth, "height": winchHeight})
continue
}
} else if command.Command == "signal" {
err := unix.Kill(cmd.Process.Pid, unix.Signal(command.Signal))
if err != nil {
l.Debug("Failed forwarding signal", logger.Ctx{"err": err, "signal": command.Signal})
continue
}
l.Info("Forwarded signal", logger.Ctx{"signal": command.Signal})
}
osHandleExecControl(control, s, ptys[0], cmd, l)
}
}()
@@ -470,7 +382,7 @@ func (s *execWs) Do(op *operations.Operation) error {
conn := s.conns[0]
s.connsLock.Unlock()
readDone, writeDone := ws.Mirror(conn, linux.NewExecWrapper(waitAttachedChildIsDead, ptys[0]))
readDone, writeDone := ws.Mirror(conn, osExecWrapper(waitAttachedChildIsDead, ptys[0]))
<-readDone
<-writeDone
@@ -503,7 +415,7 @@ func (s *execWs) Do(op *operations.Operation) error {
}
}
exitStatus, err := linux.ExitStatus(cmd.Wait())
exitStatus, err := osExitStatus(cmd.Wait())
l.Debug("Instance process stopped", logger.Ctx{"err": err, "exitStatus": exitStatus})
return finisher(exitStatus, nil)

84
cmd/incus-agent/main_agent.go
View File

@@ -9,14 +9,11 @@ import (
"os"
"os/signal"
"slices"
"strings"
"sync"
"time"
"github.com/spf13/cobra"
"golang.org/x/sys/unix"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/server/instance/instancetype"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/subprocess"
@@ -111,23 +108,12 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error {
time.Sleep(300 * time.Second)
}
reconfigureNetworkInterfaces()
osReconfigureNetworkInterfaces()
// Load the kernel driver.
if !util.PathExists("/dev/vsock") {
logger.Info("Loading vsock module")
err = linux.LoadModule("vsock")
if err != nil {
return fmt.Errorf("Unable to load the vsock kernel module: %w", err)
}
// Wait for vsock device to appear.
for i := 0; i < 5; i++ {
if !util.PathExists("/dev/vsock") {
time.Sleep(1 * time.Second)
}
}
err = osLoadModules()
if err != nil {
return err
}
// Mount shares from host.
@@ -184,9 +170,9 @@ func (c *cmdAgent) Run(cmd *cobra.Command, args []string) error {
}
}
// Cancel context when SIGTEM is received.
// Cancel context on shutdown signal.
chSignal := make(chan os.Signal, 1)
signal.Notify(chSignal, unix.SIGTERM)
signal.Notify(chSignal, osShutdownSignal)
exitStatus := 0
@@ -285,7 +271,7 @@ func (c *cmdAgent) mountHostShares() {
continue
}
err = tryMountShared(mount.Source, mount.Target, mount.FSType, mount.Options)
err = osMountShared(mount.Source, mount.Target, mount.FSType, mount.Options)
if err != nil {
logger.Infof("Failed to mount %q (Type: %q, Options: %v) to %q: %v", mount.Source, "virtiofs", mount.Options, mount.Target, err)
continue
@@ -294,59 +280,3 @@ func (c *cmdAgent) mountHostShares() {
logger.Infof("Mounted %q (Type: %q, Options: %v) to %q", mount.Source, mount.FSType, mount.Options, mount.Target)
}
}
func tryMountShared(src string, dst string, fstype string, opts []string) error {
// Convert relative mounts to absolute from / otherwise dir creation fails or mount fails.
if !strings.HasPrefix(dst, "/") {
dst = fmt.Sprintf("/%s", dst)
}
// Check mount path.
if !util.PathExists(dst) {
// Create the mount path.
err := os.MkdirAll(dst, 0o755)
if err != nil {
return fmt.Errorf("Failed to create mount target %q", dst)
}
} else if linux.IsMountPoint(dst) {
// Already mounted.
return nil
}
// Prepare the arguments.
sharedArgs := []string{}
p9Args := []string{}
for _, opt := range opts {
// transport and msize mount option are specific to 9p.
if strings.HasPrefix(opt, "trans=") || strings.HasPrefix(opt, "msize=") {
p9Args = append(p9Args, "-o", opt)
continue
}
sharedArgs = append(sharedArgs, "-o", opt)
}
// Always try virtiofs first.
args := []string{"-t", "virtiofs", src, dst}
args = append(args, sharedArgs...)
_, err := subprocess.RunCommand("mount", args...)
if err == nil {
return nil
} else if fstype == "virtiofs" {
return err
}
// Then fallback to 9p.
args = []string{"-t", "9p", src, dst}
args = append(args, sharedArgs...)
args = append(args, p9Args...)
_, err = subprocess.RunCommand("mount", args...)
if err != nil {
return err
}
return nil
}

338
cmd/incus-agent/metrics.go
View File

@@ -1,32 +1,13 @@
package main
import (
"bufio"
"bytes"
"fmt"
"net/http"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/server/metrics"
"github.com/lxc/incus/v6/internal/server/response"
"github.com/lxc/incus/v6/shared/logger"
)
// These mountpoints are excluded as they are irrelevant for metrics.
// /var/lib/docker/* subdirectories are excluded for this reason: https://github.com/prometheus/node_exporter/pull/1003
var (
defMountPointsExcluded = regexp.MustCompile(`^/(?:dev|proc|sys|var/lib/docker/.+)(?:$|/)`)
defFSTypesExcluded = []string{
"autofs", "binfmt_misc", "bpf", "cgroup", "cgroup2", "configfs", "debugfs", "devpts", "devtmpfs", "fusectl", "hugetlbfs", "iso9660", "mqueue", "nsfs", "overlay", "proc", "procfs", "pstore", "rpc_pipefs", "securityfs", "selinuxfs", "squashfs", "sysfs", "tracefs",
}
)
var metricsCmd = APIEndpoint{
Path: "metrics",
@@ -34,23 +15,27 @@ var metricsCmd = APIEndpoint{
}
func metricsGet(d *Daemon, r *http.Request) response.Response {
if !osMetricsSupported {
return response.NotFound(nil)
}
out := metrics.Metrics{}
diskStats, err := getDiskMetrics(d)
diskStats, err := osGetDiskMetrics(d)
if err != nil {
logger.Warn("Failed to get disk metrics", logger.Ctx{"err": err})
} else {
out.Disk = diskStats
}
filesystemStats, err := getFilesystemMetrics(d)
filesystemStats, err := osGetFilesystemMetrics(d)
if err != nil {
logger.Warn("Failed to get filesystem metrics", logger.Ctx{"err": err})
} else {
out.Filesystem = filesystemStats
}
memStats, err := getMemoryMetrics(d)
memStats, err := osGetMemoryMetrics(d)
if err != nil {
logger.Warn("Failed to get memory metrics", logger.Ctx{"err": err})
} else {
@@ -64,12 +49,12 @@ func metricsGet(d *Daemon, r *http.Request) response.Response {
out.Network = netStats
}
out.ProcessesTotal, err = getTotalProcesses(d)
out.ProcessesTotal, err = osGetTotalProcesses(d)
if err != nil {
logger.Warn("Failed to get total processes", logger.Ctx{"err": err})
}
cpuStats, err := getCPUMetrics(d)
cpuStats, err := osGetCPUMetrics(d)
if err != nil {
logger.Warn("Failed to get CPU metrics", logger.Ctx{"err": err})
} else {
@@ -79,313 +64,10 @@ func metricsGet(d *Daemon, r *http.Request) response.Response {
return response.SyncResponse(true, &out)
}
func getCPUMetrics(d *Daemon) ([]metrics.CPUMetrics, error) {
stats, err := os.ReadFile("/proc/stat")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/stat: %w", err)
}
out := []metrics.CPUMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(stats))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
// Only consider CPU info, skip everything else. Skip aggregated CPU stats since there will
// be stats for each individual CPU.
if !strings.HasPrefix(fields[0], "cpu") || fields[0] == "cpu" {
continue
}
// Validate the number of fields only for lines starting with "cpu".
if len(fields) < 9 {
return nil, fmt.Errorf("Invalid /proc/stat content: %q", line)
}
stats := metrics.CPUMetrics{}
stats.SecondsUser, err = strconv.ParseFloat(fields[1], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[1], err)
}
stats.SecondsUser /= 100
stats.SecondsNice, err = strconv.ParseFloat(fields[2], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[2], err)
}
stats.SecondsNice /= 100
stats.SecondsSystem, err = strconv.ParseFloat(fields[3], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.SecondsSystem /= 100
stats.SecondsIdle, err = strconv.ParseFloat(fields[4], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[4], err)
}
stats.SecondsIdle /= 100
stats.SecondsIOWait, err = strconv.ParseFloat(fields[5], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[5], err)
}
stats.SecondsIOWait /= 100
stats.SecondsIRQ, err = strconv.ParseFloat(fields[6], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[6], err)
}
stats.SecondsIRQ /= 100
stats.SecondsSoftIRQ, err = strconv.ParseFloat(fields[7], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[7], err)
}
stats.SecondsSoftIRQ /= 100
stats.SecondsSteal, err = strconv.ParseFloat(fields[8], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[8], err)
}
stats.SecondsSteal /= 100
stats.CPU = fields[0]
out = append(out, stats)
}
return out, nil
}
func getTotalProcesses(d *Daemon) (uint64, error) {
entries, err := os.ReadDir("/proc")
if err != nil {
return 0, fmt.Errorf("Failed to read dir %q: %w", "/proc", err)
}
pidCount := uint64(0)
for _, entry := range entries {
// Skip everything which isn't a directory
if !entry.IsDir() {
continue
}
name := entry.Name()
// Skip all non-PID directories
_, err := strconv.ParseUint(name, 10, 64)
if err != nil {
continue
}
cmdlinePath := filepath.Join("/proc", name, "cmdline")
cmdline, err := os.ReadFile(cmdlinePath)
if err != nil {
continue
}
if string(cmdline) == "" {
continue
}
pidCount++
}
return pidCount, nil
}
func getDiskMetrics(d *Daemon) ([]metrics.DiskMetrics, error) {
diskStats, err := os.ReadFile("/proc/diskstats")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/diskstats: %w", err)
}
out := []metrics.DiskMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(diskStats))
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
fields := strings.Fields(line)
if len(fields) < 10 {
return nil, fmt.Errorf("Invalid /proc/diskstats content: %q", line)
}
stats := metrics.DiskMetrics{}
stats.ReadsCompleted, err = strconv.ParseUint(fields[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
sectorsRead, err := strconv.ParseUint(fields[5], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.ReadBytes = sectorsRead * 512
stats.WritesCompleted, err = strconv.ParseUint(fields[7], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
sectorsWritten, err := strconv.ParseUint(fields[9], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.WrittenBytes = sectorsWritten * 512
stats.Device = fields[2]
out = append(out, stats)
}
return out, nil
}
func getFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) {
mounts, err := os.ReadFile("/proc/mounts")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/mounts: %w", err)
}
out := []metrics.FilesystemMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(mounts))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 3 {
return nil, fmt.Errorf("Invalid /proc/mounts content: %q", line)
}
// Skip uninteresting mounts
if slices.Contains(defFSTypesExcluded, fields[2]) || defMountPointsExcluded.MatchString(fields[1]) {
continue
}
stats := metrics.FilesystemMetrics{}
stats.Mountpoint = fields[1]
statfs, err := linux.StatVFS(stats.Mountpoint)
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", stats.Mountpoint, err)
}
fsType, err := linux.FSTypeToName(int32(statfs.Type))
if err == nil {
stats.FSType = fsType
}
stats.AvailableBytes = statfs.Bavail * uint64(statfs.Bsize)
stats.FreeBytes = statfs.Bfree * uint64(statfs.Bsize)
stats.SizeBytes = statfs.Blocks * uint64(statfs.Bsize)
stats.Device = fields[0]
out = append(out, stats)
}
return out, nil
}
func getMemoryMetrics(d *Daemon) (metrics.MemoryMetrics, error) {
content, err := os.ReadFile("/proc/meminfo")
if err != nil {
return metrics.MemoryMetrics{}, fmt.Errorf("Failed to read /proc/meminfo: %w", err)
}
out := metrics.MemoryMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 2 {
return metrics.MemoryMetrics{}, fmt.Errorf("Invalid /proc/meminfo content: %q", line)
}
fields[0] = strings.TrimRight(fields[0], ":")
value, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return metrics.MemoryMetrics{}, fmt.Errorf("Failed to parse %q: %w", fields[1], err)
}
// Multiply suffix (kB)
if len(fields) == 3 {
value *= 1024
}
// FIXME: Missing RSS
switch fields[0] {
case "Active":
out.ActiveBytes = value
case "Active(anon)":
out.ActiveAnonBytes = value
case "Active(file)":
out.ActiveFileBytes = value
case "Cached":
out.CachedBytes = value
case "Dirty":
out.DirtyBytes = value
case "HugePages_Free":
out.HugepagesFreeBytes = value
case "HugePages_Total":
out.HugepagesTotalBytes = value
case "Inactive":
out.InactiveBytes = value
case "Inactive(anon)":
out.InactiveAnonBytes = value
case "Inactive(file)":
out.InactiveFileBytes = value
case "Mapped":
out.MappedBytes = value
case "MemAvailable":
out.MemAvailableBytes = value
case "MemFree":
out.MemFreeBytes = value
case "MemTotal":
out.MemTotalBytes = value
case "Shmem":
out.ShmemBytes = value
case "SwapCached":
out.SwapBytes = value
case "Unevictable":
out.UnevictableBytes = value
case "Writeback":
out.WritebackBytes = value
}
}
return out, nil
}
func getNetworkMetrics(d *Daemon) ([]metrics.NetworkMetrics, error) {
out := []metrics.NetworkMetrics{}
for dev, state := range networkState() {
for dev, state := range osGetNetworkState() {
stats := metrics.NetworkMetrics{}
stats.ReceiveBytes = uint64(state.Counters.BytesReceived)

146
cmd/incus-agent/network.go
View File

@@ -2,20 +2,10 @@ package main
import (
"crypto/tls"
"encoding/json"
"errors"
"io/fs"
"net"
"os"
"path/filepath"
"sync"
"github.com/lxc/incus/v6/internal/linux"
deviceConfig "github.com/lxc/incus/v6/internal/server/device/config"
"github.com/lxc/incus/v6/internal/server/ip"
"github.com/lxc/incus/v6/internal/server/util"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/revert"
localtls "github.com/lxc/incus/v6/shared/tls"
)
@@ -60,139 +50,3 @@ func serverTLSConfig() (*tls.Config, error) {
tlsConfig := util.ServerTLSConfig(certInfo)
return tlsConfig, nil
}
// reconfigureNetworkInterfaces checks for the existence of files under NICConfigDir in the config share.
// Each file is named <device>.json and contains the Device Name, NIC Name, MTU and MAC address.
func reconfigureNetworkInterfaces() {
nicDirEntries, err := os.ReadDir(deviceConfig.NICConfigDir)
if err != nil {
// Abort if configuration folder does not exist (nothing to do), otherwise log and return.
if errors.Is(err, fs.ErrNotExist) {
return
}
logger.Error("Could not read network interface configuration directory", logger.Ctx{"err": err})
return
}
// Attempt to load the virtio_net driver in case it's not be loaded yet.
_ = linux.LoadModule("virtio_net")
// nicData is a map of MAC address to NICConfig.
nicData := make(map[string]deviceConfig.NICConfig, len(nicDirEntries))
for _, f := range nicDirEntries {
nicBytes, err := os.ReadFile(filepath.Join(deviceConfig.NICConfigDir, f.Name()))
if err != nil {
logger.Error("Could not read network interface configuration file", logger.Ctx{"err": err})
}
var conf deviceConfig.NICConfig
err = json.Unmarshal(nicBytes, &conf)
if err != nil {
logger.Error("Could not parse network interface configuration file", logger.Ctx{"err": err})
return
}
if conf.MACAddress != "" {
nicData[conf.MACAddress] = conf
}
}
// configureNIC applies any config specified for the interface based on its current MAC address.
configureNIC := func(currentNIC net.Interface) error {
reverter := revert.New()
defer reverter.Fail()
// Look for a NIC config entry for this interface based on its MAC address.
nic, ok := nicData[currentNIC.HardwareAddr.String()]
if !ok {
return nil
}
var changeName, changeMTU bool
if nic.NICName != "" && currentNIC.Name != nic.NICName {
changeName = true
}
if nic.MTU > 0 && currentNIC.MTU != int(nic.MTU) {
changeMTU = true
}
if !changeName && !changeMTU {
return nil // Nothing to do.
}
link := ip.Link{
Name: currentNIC.Name,
MTU: uint32(currentNIC.MTU),
}
err := link.SetDown()
if err != nil {
return err
}
reverter.Add(func() {
_ = link.SetUp()
})
// Apply the name from the NIC config if needed.
if changeName {
err = link.SetName(nic.NICName)
if err != nil {
return err
}
reverter.Add(func() {
err := link.SetName(currentNIC.Name)
if err != nil {
return
}
link.Name = currentNIC.Name
})
link.Name = nic.NICName
}
// Apply the MTU from the NIC config if needed.
if changeMTU {
err = link.SetMTU(nic.MTU)
if err != nil {
return err
}
link.MTU = nic.MTU
reverter.Add(func() {
err := link.SetMTU(uint32(currentNIC.MTU))
if err != nil {
return
}
link.MTU = uint32(currentNIC.MTU)
})
}
err = link.SetUp()
if err != nil {
return err
}
reverter.Success()
return nil
}
ifaces, err := net.Interfaces()
if err != nil {
logger.Error("Unable to read network interfaces", logger.Ctx{"err": err})
}
for _, iface := range ifaces {
err = configureNIC(iface)
if err != nil {
logger.Error("Unable to reconfigure network interface", logger.Ctx{"interface": iface.Name, "err": err})
}
}
}

952
cmd/incus-agent/os_linux.go Normal file
View File

@@ -0,0 +1,952 @@
//go:build linux
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"net"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"syscall"
"time"
"github.com/mdlayher/vsock"
"golang.org/x/sys/unix"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/ports"
deviceConfig "github.com/lxc/incus/v6/internal/server/device/config"
"github.com/lxc/incus/v6/internal/server/ip"
"github.com/lxc/incus/v6/internal/server/metrics"
"github.com/lxc/incus/v6/internal/version"
"github.com/lxc/incus/v6/shared/api"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/osarch"
"github.com/lxc/incus/v6/shared/revert"
"github.com/lxc/incus/v6/shared/subprocess"
"github.com/lxc/incus/v6/shared/util"
)
var (
// These mountpoints are excluded as they are irrelevant for metrics.
// /var/lib/docker/* subdirectories are excluded for this reason: https://github.com/prometheus/node_exporter/pull/1003
osMetricsExcludeMountpoints = regexp.MustCompile(`^/(?:dev|proc|sys|var/lib/docker/.+)(?:$|/)`)
osMetricsExcludeFilesystems = []string{"autofs", "binfmt_misc", "bpf", "cgroup", "cgroup2", "configfs", "debugfs", "devpts", "devtmpfs", "fusectl", "hugetlbfs", "iso9660", "mqueue", "nsfs", "overlay", "proc", "procfs", "pstore", "rpc_pipefs", "securityfs", "selinuxfs", "squashfs", "sysfs", "tracefs"}
osShutdownSignal = unix.SIGTERM
osExitStatus = linux.ExitStatus
osExecWrapper = linux.NewExecWrapper
osMetricsSupported = true
)
func osGetEnvironment() (*api.ServerEnvironment, error) {
uname, err := linux.Uname()
if err != nil {
return nil, err
}
serverName, err := os.Hostname()
if err != nil {
return nil, err
}
env := &api.ServerEnvironment{
Kernel: uname.Sysname,
KernelArchitecture: uname.Machine,
KernelVersion: uname.Release,
Server: "incus-agent",
ServerPid: os.Getpid(),
ServerVersion: version.Version,
ServerName: serverName,
}
return env, nil
}
func osLoadModules() error {
// Attempt to load the virtio_net driver in case it's not be loaded yet.
// This may be needed for later network configuration.
_ = linux.LoadModule("virtio_net")
// Load the vsock driver if not loaded yet, this is required for host communication.
if !util.PathExists("/dev/vsock") {
logger.Info("Loading vsock module")
err := linux.LoadModule("vsock")
if err != nil {
return fmt.Errorf("Unable to load the vsock kernel module: %w", err)
}
// Wait for vsock device to appear.
for i := 0; i < 5; i++ {
if !util.PathExists("/dev/vsock") {
time.Sleep(1 * time.Second)
}
}
}
return nil
}
func osMountShared(src string, dst string, fstype string, opts []string) error {
// Convert relative mounts to absolute from / otherwise dir creation fails or mount fails.
if !strings.HasPrefix(dst, "/") {
dst = fmt.Sprintf("/%s", dst)
}
// Check mount path.
if !util.PathExists(dst) {
// Create the mount path.
err := os.MkdirAll(dst, 0o755)
if err != nil {
return fmt.Errorf("Failed to create mount target %q", dst)
}
} else if linux.IsMountPoint(dst) {
// Already mounted.
return nil
}
// Prepare the arguments.
sharedArgs := []string{}
p9Args := []string{}
for _, opt := range opts {
// transport and msize mount option are specific to 9p.
if strings.HasPrefix(opt, "trans=") || strings.HasPrefix(opt, "msize=") {
p9Args = append(p9Args, "-o", opt)
continue
}
sharedArgs = append(sharedArgs, "-o", opt)
}
// Always try virtiofs first.
args := []string{"-t", "virtiofs", src, dst}
args = append(args, sharedArgs...)
_, err := subprocess.RunCommand("mount", args...)
if err == nil {
return nil
} else if fstype == "virtiofs" {
return err
}
// Then fallback to 9p.
args = []string{"-t", "9p", src, dst}
args = append(args, sharedArgs...)
args = append(args, p9Args...)
_, err = subprocess.RunCommand("mount", args...)
if err != nil {
return err
}
return nil
}
func osGetCPUMetrics(d *Daemon) ([]metrics.CPUMetrics, error) {
stats, err := os.ReadFile("/proc/stat")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/stat: %w", err)
}
out := []metrics.CPUMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(stats))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
// Only consider CPU info, skip everything else. Skip aggregated CPU stats since there will
// be stats for each individual CPU.
if !strings.HasPrefix(fields[0], "cpu") || fields[0] == "cpu" {
continue
}
// Validate the number of fields only for lines starting with "cpu".
if len(fields) < 9 {
return nil, fmt.Errorf("Invalid /proc/stat content: %q", line)
}
stats := metrics.CPUMetrics{}
stats.SecondsUser, err = strconv.ParseFloat(fields[1], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[1], err)
}
stats.SecondsUser /= 100
stats.SecondsNice, err = strconv.ParseFloat(fields[2], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[2], err)
}
stats.SecondsNice /= 100
stats.SecondsSystem, err = strconv.ParseFloat(fields[3], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.SecondsSystem /= 100
stats.SecondsIdle, err = strconv.ParseFloat(fields[4], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[4], err)
}
stats.SecondsIdle /= 100
stats.SecondsIOWait, err = strconv.ParseFloat(fields[5], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[5], err)
}
stats.SecondsIOWait /= 100
stats.SecondsIRQ, err = strconv.ParseFloat(fields[6], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[6], err)
}
stats.SecondsIRQ /= 100
stats.SecondsSoftIRQ, err = strconv.ParseFloat(fields[7], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[7], err)
}
stats.SecondsSoftIRQ /= 100
stats.SecondsSteal, err = strconv.ParseFloat(fields[8], 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[8], err)
}
stats.SecondsSteal /= 100
stats.CPU = fields[0]
out = append(out, stats)
}
return out, nil
}
func osGetTotalProcesses(d *Daemon) (uint64, error) {
entries, err := os.ReadDir("/proc")
if err != nil {
return 0, fmt.Errorf("Failed to read dir %q: %w", "/proc", err)
}
pidCount := uint64(0)
for _, entry := range entries {
// Skip everything which isn't a directory
if !entry.IsDir() {
continue
}
name := entry.Name()
// Skip all non-PID directories
_, err := strconv.ParseUint(name, 10, 64)
if err != nil {
continue
}
cmdlinePath := filepath.Join("/proc", name, "cmdline")
cmdline, err := os.ReadFile(cmdlinePath)
if err != nil {
continue
}
if string(cmdline) == "" {
continue
}
pidCount++
}
return pidCount, nil
}
func osGetDiskMetrics(d *Daemon) ([]metrics.DiskMetrics, error) {
diskStats, err := os.ReadFile("/proc/diskstats")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/diskstats: %w", err)
}
out := []metrics.DiskMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(diskStats))
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
fields := strings.Fields(line)
if len(fields) < 10 {
return nil, fmt.Errorf("Invalid /proc/diskstats content: %q", line)
}
stats := metrics.DiskMetrics{}
stats.ReadsCompleted, err = strconv.ParseUint(fields[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
sectorsRead, err := strconv.ParseUint(fields[5], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.ReadBytes = sectorsRead * 512
stats.WritesCompleted, err = strconv.ParseUint(fields[7], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
sectorsWritten, err := strconv.ParseUint(fields[9], 10, 64)
if err != nil {
return nil, fmt.Errorf("Failed to parse %q: %w", fields[3], err)
}
stats.WrittenBytes = sectorsWritten * 512
stats.Device = fields[2]
out = append(out, stats)
}
return out, nil
}
func osGetFilesystemMetrics(d *Daemon) ([]metrics.FilesystemMetrics, error) {
mounts, err := os.ReadFile("/proc/mounts")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/mounts: %w", err)
}
out := []metrics.FilesystemMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(mounts))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 3 {
return nil, fmt.Errorf("Invalid /proc/mounts content: %q", line)
}
// Skip uninteresting mounts
if slices.Contains(osMetricsExcludeFilesystems, fields[2]) || osMetricsExcludeMountpoints.MatchString(fields[1]) {
continue
}
stats := metrics.FilesystemMetrics{}
stats.Mountpoint = fields[1]
statfs, err := linux.StatVFS(stats.Mountpoint)
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", stats.Mountpoint, err)
}
fsType, err := linux.FSTypeToName(int32(statfs.Type))
if err == nil {
stats.FSType = fsType
}
stats.AvailableBytes = statfs.Bavail * uint64(statfs.Bsize)
stats.FreeBytes = statfs.Bfree * uint64(statfs.Bsize)
stats.SizeBytes = statfs.Blocks * uint64(statfs.Bsize)
stats.Device = fields[0]
out = append(out, stats)
}
return out, nil
}
func osGetMemoryMetrics(d *Daemon) (metrics.MemoryMetrics, error) {
content, err := os.ReadFile("/proc/meminfo")
if err != nil {
return metrics.MemoryMetrics{}, fmt.Errorf("Failed to read /proc/meminfo: %w", err)
}
out := metrics.MemoryMetrics{}
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
line := scanner.Text()
fields := strings.Fields(line)
if len(fields) < 2 {
return metrics.MemoryMetrics{}, fmt.Errorf("Invalid /proc/meminfo content: %q", line)
}
fields[0] = strings.TrimRight(fields[0], ":")
value, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return metrics.MemoryMetrics{}, fmt.Errorf("Failed to parse %q: %w", fields[1], err)
}
// Multiply suffix (kB)
if len(fields) == 3 {
value *= 1024
}
// FIXME: Missing RSS
switch fields[0] {
case "Active":
out.ActiveBytes = value
case "Active(anon)":
out.ActiveAnonBytes = value
case "Active(file)":
out.ActiveFileBytes = value
case "Cached":
out.CachedBytes = value
case "Dirty":
out.DirtyBytes = value
case "HugePages_Free":
out.HugepagesFreeBytes = value
case "HugePages_Total":
out.HugepagesTotalBytes = value
case "Inactive":
out.InactiveBytes = value
case "Inactive(anon)":
out.InactiveAnonBytes = value
case "Inactive(file)":
out.InactiveFileBytes = value
case "Mapped":
out.MappedBytes = value
case "MemAvailable":
out.MemAvailableBytes = value
case "MemFree":
out.MemFreeBytes = value
case "MemTotal":
out.MemTotalBytes = value
case "Shmem":
out.ShmemBytes = value
case "SwapCached":
out.SwapBytes = value
case "Unevictable":
out.UnevictableBytes = value
case "Writeback":
out.WritebackBytes = value
}
}
return out, nil
}
func osGetCPUState() api.InstanceStateCPU {
var value []byte
var err error
cpu := api.InstanceStateCPU{}
if util.PathExists("/sys/fs/cgroup/cpuacct/cpuacct.usage") {
// CPU usage in seconds
value, err = os.ReadFile("/sys/fs/cgroup/cpuacct/cpuacct.usage")
if err != nil {
cpu.Usage = -1
return cpu
}
valueInt, err := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err != nil {
cpu.Usage = -1
return cpu
}
cpu.Usage = valueInt
return cpu
} else if util.PathExists("/sys/fs/cgroup/cpu.stat") {
stats, err := os.ReadFile("/sys/fs/cgroup/cpu.stat")
if err != nil {
cpu.Usage = -1
return cpu
}
scanner := bufio.NewScanner(bytes.NewReader(stats))
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if fields[0] == "usage_usec" {
valueInt, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil {
cpu.Usage = -1
return cpu
}
// usec -> nsec
cpu.Usage = valueInt * 1000
return cpu
}
}
}
cpu.Usage = -1
return cpu
}
func osGetMemoryState() api.InstanceStateMemory {
memory := api.InstanceStateMemory{}
stats, err := osGetMemoryMetrics(nil)
if err != nil {
return memory
}
memory.Usage = int64(stats.MemTotalBytes) - int64(stats.MemFreeBytes)
memory.Total = int64(stats.MemTotalBytes)
// Memory peak in bytes
value, err := os.ReadFile("/sys/fs/cgroup/memory/memory.max_usage_in_bytes")
valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
memory.UsagePeak = valueInt
}
return memory
}
func osGetNetworkState() map[string]api.InstanceStateNetwork {
result := map[string]api.InstanceStateNetwork{}
ifs, err := linux.NetlinkInterfaces()
if err != nil {
logger.Errorf("Failed to retrieve network interfaces: %v", err)
return result
}
for _, iface := range ifs {
network := api.InstanceStateNetwork{
Addresses: []api.InstanceStateNetworkAddress{},
Counters: api.InstanceStateNetworkCounters{},
}
network.Hwaddr = iface.HardwareAddr.String()
network.Mtu = iface.MTU
if iface.Flags&net.FlagUp != 0 {
network.State = "up"
} else {
network.State = "down"
}
if iface.Flags&net.FlagBroadcast != 0 {
network.Type = "broadcast"
} else if iface.Flags&net.FlagLoopback != 0 {
network.Type = "loopback"
} else if iface.Flags&net.FlagPointToPoint != 0 {
network.Type = "point-to-point"
} else {
network.Type = "unknown"
}
// Counters
value, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_bytes", iface.Name))
valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.BytesSent = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_bytes", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.BytesReceived = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_packets", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.PacketsSent = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_packets", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.PacketsReceived = valueInt
}
// Addresses
for _, addr := range iface.Addresses {
addressFields := strings.Split(addr.String(), "/")
networkAddress := api.InstanceStateNetworkAddress{
Address: addressFields[0],
Netmask: addressFields[1],
}
scope := "global"
if strings.HasPrefix(addressFields[0], "127") {
scope = "local"
}
if addressFields[0] == "::1" {
scope = "local"
}
if strings.HasPrefix(addressFields[0], "169.254") {
scope = "link"
}
if strings.HasPrefix(addressFields[0], "fe80:") {
scope = "link"
}
networkAddress.Scope = scope
if strings.Contains(addressFields[0], ":") {
networkAddress.Family = "inet6"
} else {
networkAddress.Family = "inet"
}
network.Addresses = append(network.Addresses, networkAddress)
}
result[iface.Name] = network
}
return result
}
func osGetProcessesState() int64 {
pids := []int64{1}
// Go through the pid list, adding new pids at the end so we go through them all
for i := 0; i < len(pids); i++ {
fname := fmt.Sprintf("/proc/%d/task/%d/children", pids[i], pids[i])
fcont, err := os.ReadFile(fname)
if err != nil {
// the process terminated during execution of this loop
continue
}
content := strings.Split(string(fcont), " ")
for j := 0; j < len(content); j++ {
pid, err := strconv.ParseInt(content[j], 10, 64)
if err == nil {
pids = append(pids, pid)
}
}
}
return int64(len(pids))
}
func osGetOSState() *api.InstanceStateOSInfo {
osInfo := &api.InstanceStateOSInfo{}
// Get information about the OS.
lsbRelease, err := osarch.GetOSRelease()
if err == nil {
osInfo.OS = lsbRelease["NAME"]
osInfo.OSVersion = lsbRelease["VERSION_ID"]
}
// Get information about the kernel version.
uname, err := linux.Uname()
if err == nil {
osInfo.KernelVersion = uname.Release
}
// Get the hostname.
hostname, err := os.Hostname()
if err == nil {
osInfo.Hostname = hostname
}
// Get the FQDN. To avoid needing to run `hostname -f`, do a reverse host lookup for 127.0.1.1, and if found, return the first hostname as the FQDN.
ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
defer cancel()
var r net.Resolver
fqdn, err := r.LookupAddr(ctx, "127.0.0.1")
if err == nil && len(fqdn) > 0 {
// Take the first returned hostname and trim the trailing dot.
osInfo.FQDN = strings.TrimSuffix(fqdn[0], ".")
}
return osInfo
}
// osReconfigureNetworkInterfaces checks for the existence of files under NICConfigDir in the config share.
// Each file is named <device>.json and contains the Device Name, NIC Name, MTU and MAC address.
func osReconfigureNetworkInterfaces() {
nicDirEntries, err := os.ReadDir(deviceConfig.NICConfigDir)
if err != nil {
// Abort if configuration folder does not exist (nothing to do), otherwise log and return.
if errors.Is(err, fs.ErrNotExist) {
return
}
logger.Error("Could not read network interface configuration directory", logger.Ctx{"err": err})
return
}
// Attempt to load the virtio_net driver in case it's not be loaded yet.
_ = linux.LoadModule("virtio_net")
// nicData is a map of MAC address to NICConfig.
nicData := make(map[string]deviceConfig.NICConfig, len(nicDirEntries))
for _, f := range nicDirEntries {
nicBytes, err := os.ReadFile(filepath.Join(deviceConfig.NICConfigDir, f.Name()))
if err != nil {
logger.Error("Could not read network interface configuration file", logger.Ctx{"err": err})
}
var conf deviceConfig.NICConfig
err = json.Unmarshal(nicBytes, &conf)
if err != nil {
logger.Error("Could not parse network interface configuration file", logger.Ctx{"err": err})
return
}
if conf.MACAddress != "" {
nicData[conf.MACAddress] = conf
}
}
// configureNIC applies any config specified for the interface based on its current MAC address.
configureNIC := func(currentNIC net.Interface) error {
reverter := revert.New()
defer reverter.Fail()
// Look for a NIC config entry for this interface based on its MAC address.
nic, ok := nicData[currentNIC.HardwareAddr.String()]
if !ok {
return nil
}
var changeName, changeMTU bool
if nic.NICName != "" && currentNIC.Name != nic.NICName {
changeName = true
}
if nic.MTU > 0 && currentNIC.MTU != int(nic.MTU) {
changeMTU = true
}
if !changeName && !changeMTU {
return nil // Nothing to do.
}
link := ip.Link{
Name: currentNIC.Name,
MTU: uint32(currentNIC.MTU),
}
err := link.SetDown()
if err != nil {
return err
}
reverter.Add(func() {
_ = link.SetUp()
})
// Apply the name from the NIC config if needed.
if changeName {
err = link.SetName(nic.NICName)
if err != nil {
return err
}
reverter.Add(func() {
err := link.SetName(currentNIC.Name)
if err != nil {
return
}
link.Name = currentNIC.Name
})
link.Name = nic.NICName
}
// Apply the MTU from the NIC config if needed.
if changeMTU {
err = link.SetMTU(nic.MTU)
if err != nil {
return err
}
link.MTU = nic.MTU
reverter.Add(func() {
err := link.SetMTU(uint32(currentNIC.MTU))
if err != nil {
return
}
link.MTU = uint32(currentNIC.MTU)
})
}
err = link.SetUp()
if err != nil {
return err
}
reverter.Success()
return nil
}
ifaces, err := net.Interfaces()
if err != nil {
logger.Error("Unable to read network interfaces", logger.Ctx{"err": err})
}
for _, iface := range ifaces {
err = configureNIC(iface)
if err != nil {
logger.Error("Unable to reconfigure network interface", logger.Ctx{"interface": iface.Name, "err": err})
}
}
}
func osGetInteractiveConsole(s *execWs) (*os.File, *os.File, error) {
pty, tty, err := linux.OpenPty(int64(s.uid), int64(s.gid))
if err != nil {
return nil, nil, err
}
if s.width > 0 && s.height > 0 {
_ = linux.SetPtySize(int(pty.Fd()), s.width, s.height)
}
return pty, tty, nil
}
func osPrepareExecCommand(s *execWs, cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{
Credential: &syscall.Credential{
Uid: s.uid,
Gid: s.gid,
},
// Creates a new session if the calling process is not a process group leader.
// The calling process is the leader of the new session, the process group leader of
// the new process group, and has no controlling terminal.
// This is important to allow remote shells to handle ctrl+c.
Setsid: true,
}
// Make the given terminal the controlling terminal of the calling process.
// The calling process must be a session leader and not have a controlling terminal already.
// This is important as allows ctrl+c to work as expected for non-shell programs.
if s.interactive {
cmd.SysProcAttr.Setctty = true
}
}
func osHandleExecControl(control api.InstanceExecControl, s *execWs, pty *os.File, cmd *exec.Cmd, l logger.Logger) {
if control.Command == "window-resize" && s.interactive {
winchWidth, err := strconv.Atoi(control.Args["width"])
if err != nil {
l.Debug("Unable to extract window width", logger.Ctx{"err": err})
return
}
winchHeight, err := strconv.Atoi(control.Args["height"])
if err != nil {
l.Debug("Unable to extract window height", logger.Ctx{"err": err})
return
}
err = linux.SetPtySize(int(pty.Fd()), winchWidth, winchHeight)
if err != nil {
l.Debug("Failed to set window size", logger.Ctx{"err": err, "width": winchWidth, "height": winchHeight})
return
}
} else if control.Command == "signal" {
err := unix.Kill(cmd.Process.Pid, unix.Signal(control.Signal))
if err != nil {
l.Debug("Failed forwarding signal", logger.Ctx{"err": err, "signal": control.Signal})
return
}
l.Info("Forwarded signal", logger.Ctx{"signal": control.Signal})
}
}
func osGetListener(port int64) (net.Listener, error) {
const CIDAny uint32 = 4294967295 // Equivalent to VMADDR_CID_ANY.
// Setup the listener on wildcard CID for inbound connections from Incus.
// We use the VMADDR_CID_ANY CID so that if the VM's CID changes in the future the listener still works.
// A CID change can occur when restoring a stateful VM that was previously using one CID but is
// subsequently restored using a different one.
l, err := vsock.ListenContextID(CIDAny, ports.HTTPSDefaultPort, nil)
if err != nil {
return nil, fmt.Errorf("Failed to listen on vsock: %w", err)
}
logger.Info("Started vsock listener")
return l, nil
}
func osSetEnv(post *api.InstanceExecPost, env map[string]string) {
// Set default value for PATH.
_, ok := env["PATH"]
if !ok {
env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
}
if util.PathExists("/snap/bin") {
env["PATH"] = fmt.Sprintf("%s:/snap/bin", env["PATH"])
}
// If running as root, set some env variables.
if post.User == 0 {
// Set default value for HOME.
_, ok = env["HOME"]
if !ok {
env["HOME"] = "/root"
}
// Set default value for USER.
_, ok = env["USER"]
if !ok {
env["USER"] = "root"
}
}
// Set default value for LANG.
_, ok = env["LANG"]
if !ok {
env["LANG"] = "C.UTF-8"
}
// Set the default working directory.
if post.Cwd == "" {
post.Cwd = env["HOME"]
if post.Cwd == "" {
post.Cwd = "/"
}
}
}

258
cmd/incus-agent/state.go
View File

@@ -1,23 +1,10 @@
package main
import (
"bufio"
"bytes"
"context"
"fmt"
"net"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/lxc/incus/v6/internal/linux"
"github.com/lxc/incus/v6/internal/server/response"
"github.com/lxc/incus/v6/shared/api"
"github.com/lxc/incus/v6/shared/logger"
"github.com/lxc/incus/v6/shared/osarch"
"github.com/lxc/incus/v6/shared/util"
)
var stateCmd = APIEndpoint{
@@ -38,246 +25,11 @@ func statePut(d *Daemon, r *http.Request) response.Response {
func renderState() *api.InstanceState {
return &api.InstanceState{
CPU: cpuState(),
Memory: memoryState(),
Network: networkState(),
CPU: osGetCPUState(),
Memory: osGetMemoryState(),
Network: osGetNetworkState(),
Pid: 1,
Processes: processesState(),
OSInfo: osState(),
Processes: osGetProcessesState(),
OSInfo: osGetOSState(),
}
}
func cpuState() api.InstanceStateCPU {
var value []byte
var err error
cpu := api.InstanceStateCPU{}
if util.PathExists("/sys/fs/cgroup/cpuacct/cpuacct.usage") {
// CPU usage in seconds
value, err = os.ReadFile("/sys/fs/cgroup/cpuacct/cpuacct.usage")
if err != nil {
cpu.Usage = -1
return cpu
}
valueInt, err := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err != nil {
cpu.Usage = -1
return cpu
}
cpu.Usage = valueInt
return cpu
} else if util.PathExists("/sys/fs/cgroup/cpu.stat") {
stats, err := os.ReadFile("/sys/fs/cgroup/cpu.stat")
if err != nil {
cpu.Usage = -1
return cpu
}
scanner := bufio.NewScanner(bytes.NewReader(stats))
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if fields[0] == "usage_usec" {
valueInt, err := strconv.ParseInt(fields[1], 10, 64)
if err != nil {
cpu.Usage = -1
return cpu
}
// usec -> nsec
cpu.Usage = valueInt * 1000
return cpu
}
}
}
cpu.Usage = -1
return cpu
}
func memoryState() api.InstanceStateMemory {
memory := api.InstanceStateMemory{}
stats, err := getMemoryMetrics(nil)
if err != nil {
return memory
}
memory.Usage = int64(stats.MemTotalBytes) - int64(stats.MemFreeBytes)
memory.Total = int64(stats.MemTotalBytes)
// Memory peak in bytes
value, err := os.ReadFile("/sys/fs/cgroup/memory/memory.max_usage_in_bytes")
valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
memory.UsagePeak = valueInt
}
return memory
}
func networkState() map[string]api.InstanceStateNetwork {
result := map[string]api.InstanceStateNetwork{}
ifs, err := linux.NetlinkInterfaces()
if err != nil {
logger.Errorf("Failed to retrieve network interfaces: %v", err)
return result
}
for _, iface := range ifs {
network := api.InstanceStateNetwork{
Addresses: []api.InstanceStateNetworkAddress{},
Counters: api.InstanceStateNetworkCounters{},
}
network.Hwaddr = iface.HardwareAddr.String()
network.Mtu = iface.MTU
if iface.Flags&net.FlagUp != 0 {
network.State = "up"
} else {
network.State = "down"
}
if iface.Flags&net.FlagBroadcast != 0 {
network.Type = "broadcast"
} else if iface.Flags&net.FlagLoopback != 0 {
network.Type = "loopback"
} else if iface.Flags&net.FlagPointToPoint != 0 {
network.Type = "point-to-point"
} else {
network.Type = "unknown"
}
// Counters
value, err := os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_bytes", iface.Name))
valueInt, err1 := strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.BytesSent = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_bytes", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.BytesReceived = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/tx_packets", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.PacketsSent = valueInt
}
value, err = os.ReadFile(fmt.Sprintf("/sys/class/net/%s/statistics/rx_packets", iface.Name))
valueInt, err1 = strconv.ParseInt(strings.TrimSpace(string(value)), 10, 64)
if err == nil && err1 == nil {
network.Counters.PacketsReceived = valueInt
}
// Addresses
for _, addr := range iface.Addresses {
addressFields := strings.Split(addr.String(), "/")
networkAddress := api.InstanceStateNetworkAddress{
Address: addressFields[0],
Netmask: addressFields[1],
}
scope := "global"
if strings.HasPrefix(addressFields[0], "127") {
scope = "local"
}
if addressFields[0] == "::1" {
scope = "local"
}
if strings.HasPrefix(addressFields[0], "169.254") {
scope = "link"
}
if strings.HasPrefix(addressFields[0], "fe80:") {
scope = "link"
}
networkAddress.Scope = scope
if strings.Contains(addressFields[0], ":") {
networkAddress.Family = "inet6"
} else {
networkAddress.Family = "inet"
}
network.Addresses = append(network.Addresses, networkAddress)
}
result[iface.Name] = network
}
return result
}
func processesState() int64 {
pids := []int64{1}
// Go through the pid list, adding new pids at the end so we go through them all
for i := 0; i < len(pids); i++ {
fname := fmt.Sprintf("/proc/%d/task/%d/children", pids[i], pids[i])
fcont, err := os.ReadFile(fname)
if err != nil {
// the process terminated during execution of this loop
continue
}
content := strings.Split(string(fcont), " ")
for j := 0; j < len(content); j++ {
pid, err := strconv.ParseInt(content[j], 10, 64)
if err == nil {
pids = append(pids, pid)
}
}
}
return int64(len(pids))
}
func osState() *api.InstanceStateOSInfo {
osInfo := &api.InstanceStateOSInfo{}
// Get information about the OS.
lsbRelease, err := osarch.GetOSRelease()
if err == nil {
osInfo.OS = lsbRelease["NAME"]
osInfo.OSVersion = lsbRelease["VERSION_ID"]
}
// Get information about the kernel version.
uname, err := linux.Uname()
if err == nil {
osInfo.KernelVersion = uname.Release
}
// Get the hostname.
hostname, err := os.Hostname()
if err == nil {
osInfo.Hostname = hostname
}
// Get the FQDN. To avoid needing to run `hostname -f`, do a reverse host lookup for 127.0.1.1, and if found, return the first hostname as the FQDN.
ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
defer cancel()
var r net.Resolver
fqdn, err := r.LookupAddr(ctx, "127.0.0.1")
if err == nil && len(fqdn) > 0 {
// Take the first returned hostname and trim the trailing dot.
osInfo.FQDN = strings.TrimSuffix(fqdn[0], ".")
}
return osInfo
}