prometheus-operator/test/framework/context.go

// Copyright 2017 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"golang.org/x/sync/errgroup"
	"gopkg.in/yaml.v2"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
)

type TestCtx struct {
	id         string
	namespaces []string
	cleanUpFns []FinalizerFn
}

type FinalizerFn func() error

type diagnosticWriter interface {
	io.Writer
	io.Closer
	StartCollection(string)
}

// stdoutDiagnosticWriter writes collected information to stdout.
type stdoutDiagnosticWriter struct {
	b bytes.Buffer
}

func (sdw *stdoutDiagnosticWriter) Write(b []byte) (int, error) { return sdw.b.Write(b) }
func (sdw *stdoutDiagnosticWriter) Close() error                { return nil }
func (sdw *stdoutDiagnosticWriter) StartCollection(name string) {
	fmt.Fprintf(&sdw.b, "=== %s\n", name)
}

// fileDiagnosticWriter writes collected information to disk.
type fileDiagnosticWriter struct {
	dir string
	f   *os.File
}

func (fdw *fileDiagnosticWriter) Write(b []byte) (int, error) {
	if fdw.f == nil {
		return 0, nil
	}

	return fdw.f.Write(b)
}

func (fdw *fileDiagnosticWriter) Close() error {
	if fdw.f == nil {
		return nil
	}

	return fdw.f.Close()
}

func (fdw *fileDiagnosticWriter) StartCollection(name string) {
	if fdw.f != nil {
		fdw.f.Close()
		fdw.f = nil
	}

	fullpath := filepath.Join(fdw.dir, name)
	if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil {
		return
	}

	f, err := os.Create(fullpath)
	if err != nil {
		return
	}

	fdw.f = f
}

func (f *Framework) NewTestCtx(t *testing.T) *TestCtx {
	// TestCtx is used among others for namespace names where '/' is forbidden
	prefix := strings.TrimPrefix(
		strings.ReplaceAll(
			strings.ToLower(t.Name()),
			"/",
			"-",
		),
		"test",
	)

	tc := &TestCtx{
		id: prefix + "-" + strconv.FormatInt(time.Now().Unix(), 36),
	}

	tc.cleanUpFns = []FinalizerFn{
		func() error {
			t.Helper()
			if !t.Failed() {
				return nil
			}

			// We can collect more information as we see fit over time.
			var (
				dw  diagnosticWriter
				dir = os.Getenv("E2E_DIAGNOSTIC_DIRECTORY")
			)

			var verbose bool
			if dir != "" {
				dw = &fileDiagnosticWriter{
					dir: filepath.Join(dir, t.Name()),
				}
				verbose = true
			} else {
				dw = &stdoutDiagnosticWriter{}
			}
			defer dw.Close()

			// Workload resources
			dw.StartCollection("alertmanagers")
			tc.collectAlertmanagers(dw, f, verbose)
			dw.StartCollection("prometheuses")
			tc.collectPrometheuses(dw, f, verbose)
			dw.StartCollection("thanosrulers")
			tc.collectThanosRulers(dw, f, verbose)
			dw.StartCollection("prometheusagents")
			tc.collectPrometheusAgents(dw, f, verbose)

			// Configuration resources
			dw.StartCollection("servicemonitors")
			tc.collectServiceMonitors(dw, f, verbose)
			dw.StartCollection("podmonitors")
			tc.collectPodMonitors(dw, f, verbose)
			dw.StartCollection("probes")
			tc.collectProbes(dw, f, verbose)
			dw.StartCollection("prometheusrules")
			tc.collectPrometheusRules(dw, f, verbose)
			dw.StartCollection("scrapeconfigs")
			tc.collectScrapeConfigs(dw, f, verbose)
			dw.StartCollection("alertmanagerconfigs")
			tc.collectAlertmanagerConfigs(dw, f, verbose)

			// Kubernetes resources
			dw.StartCollection("statefulsets")
			tc.collectStatefulSets(dw, f, verbose)
			dw.StartCollection("daemonsets")
			tc.collectDaemonSets(dw, f, verbose)
			dw.StartCollection("pods")
			tc.collectPods(dw, f, verbose)
			dw.StartCollection("services")
			tc.collectServices(dw, f, verbose)
			dw.StartCollection("configmaps")
			tc.collectConfigMaps(dw, f, verbose)
			dw.StartCollection("secrets")
			tc.collectSecrets(dw, f, verbose)

			tc.collectLogs(dw, f)

			tc.collectEvents(dw, f)

			if sdw, ok := dw.(*stdoutDiagnosticWriter); ok {
				t.Logf("== %s (start)", t.Name())
				t.Log("")
				t.Log(sdw.b.String())
				t.Logf("== %s (end)", t.Name())
			}

			return nil
		},
	}

	return tc
}

func (ctx *TestCtx) collectLogs(dw diagnosticWriter, f *Framework) {
	for _, ns := range ctx.namespaces {
		pods, err := f.KubeClient.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: failed to get pods: %v\n", ns, err)
			continue
		}

		for _, pod := range pods.Items {
			dw.StartCollection(filepath.Join("logs", pod.Namespace, pod.Name))
			err := f.WritePodLogs(context.Background(), dw, ns, pod.Name, LogOptions{})
			if err != nil {
				fmt.Fprintf(os.Stderr, "%s: failed to get pod logs: %v\n", ns, err)
				continue
			}
		}
	}
}

func (ctx *TestCtx) collectEvents(dw diagnosticWriter, f *Framework) {
	for _, ns := range ctx.namespaces {
		dw.StartCollection(filepath.Join("events", ns))
		err := f.WriteEvents(context.Background(), dw, ns)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: failed to get events: %v\n", ns, err)
		}
	}
}

func collectConditions(w io.Writer, prefix string, conditions []monitoringv1.Condition) {
	for _, c := range conditions {
		fmt.Fprintf(
			w,
			"%s: condition type=%q status=%q reason=%q message=%q\n",
			prefix,
			c.Type,
			c.Status,
			c.Reason,
			c.Message,
		)
	}
}

func writeYAML(w io.Writer, res any) {
	enc := yaml.NewEncoder(w)
	_ = enc.Encode(res)
	_ = enc.Close()
}

func collectBindingConditions(w io.Writer, prefix string, bindings []monitoringv1.WorkloadBinding) {
	for _, b := range bindings {
		for _, c := range b.Conditions {
			fmt.Fprintf(
				w,
				"%s: binding %s/%s/%s: condition type=%q status=%q reason=%q message=%q\n",
				prefix,
				b.Resource,
				b.Namespace,
				b.Name,
				c.Type,
				c.Status,
				c.Reason,
				c.Message,
			)
		}
	}
}

func (ctx *TestCtx) collectAlertmanagers(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		ams, err := f.MonClientV1.Alertmanagers(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get alertmanagers: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, ams)
			continue
		}

		for _, am := range ams.Items {
			collectConditions(w, fmt.Sprintf("Alertmanager=%s/%s", am.Namespace, am.Name), am.Status.Conditions)
		}
	}
}

func (ctx *TestCtx) collectPrometheuses(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		ps, err := f.MonClientV1.Prometheuses(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get prometheuses: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, ps)
			continue
		}

		for _, p := range ps.Items {
			collectConditions(w, fmt.Sprintf("Prometheus=%s/%s", p.Namespace, p.Name), p.Status.Conditions)
		}
	}
}

func (ctx *TestCtx) collectPrometheusAgents(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		ps, err := f.MonClientV1alpha1.PrometheusAgents(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get prometheusagents: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, ps)
			continue
		}

		for _, p := range ps.Items {
			collectConditions(w, fmt.Sprintf("PrometheusAgent=%s/%s", p.Namespace, p.Name), p.Status.Conditions)
		}
	}
}

func (ctx *TestCtx) collectThanosRulers(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		trs, err := f.MonClientV1.ThanosRulers(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get thanosrulers: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, trs)
			continue
		}

		for _, tr := range trs.Items {
			collectConditions(w, fmt.Sprintf("ThanosRuler=%s/%s", tr.Namespace, tr.Name), tr.Status.Conditions)
		}
	}
}

func (ctx *TestCtx) collectServiceMonitors(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		sms, err := f.MonClientV1.ServiceMonitors(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get servicemonitors: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, sms)
			continue
		}

		for _, sm := range sms.Items {
			collectBindingConditions(w, fmt.Sprintf("ServiceMonitor=%s/%s", sm.Namespace, sm.Name), sm.Status.Bindings)
		}
	}
}

func (ctx *TestCtx) collectPodMonitors(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		pms, err := f.MonClientV1.PodMonitors(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get podmonitors: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, pms)
			continue
		}

		for _, pm := range pms.Items {
			collectBindingConditions(w, fmt.Sprintf("PodMonitor=%s/%s", pm.Namespace, pm.Name), pm.Status.Bindings)
		}
	}
}

func (ctx *TestCtx) collectProbes(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		ps, err := f.MonClientV1.Probes(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get probes: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, ps)
			continue
		}

		for _, p := range ps.Items {
			//TODO(simonpasquier): provide workload bindings when implemented.
			collectBindingConditions(w, fmt.Sprintf("Probe=%s/%s", p.Namespace, p.Name), nil)
		}
	}
}

func (ctx *TestCtx) collectPrometheusRules(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		prs, err := f.MonClientV1.PrometheusRules(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get prometheusrules: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, prs)
			continue
		}

		for _, pr := range prs.Items {
			//TODO(simonpasquier): provide workload bindings when implemented.
			collectBindingConditions(w, fmt.Sprintf("PrometheusRule=%s/%s", pr.Namespace, pr.Name), nil)
		}
	}
}

func (ctx *TestCtx) collectScrapeConfigs(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		scs, err := f.MonClientV1alpha1.ScrapeConfigs(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get scrapeconfigs: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, scs)
			continue
		}

		for _, sc := range scs.Items {
			//TODO(simonpasquier): provide workload bindings when implemented.
			collectBindingConditions(w, fmt.Sprintf("ScrapeConfig=%s/%s", sc.Namespace, sc.Name), nil)
		}
	}
}

func (ctx *TestCtx) collectAlertmanagerConfigs(w io.Writer, f *Framework, verbose bool) {
	for _, ns := range ctx.namespaces {
		acs, err := f.MonClientV1beta1.AlertmanagerConfigs(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get alertmanagerconfigs: %v\n", ns, err)
			continue
		}

		if verbose {
			writeYAML(w, acs)
			continue
		}

		for _, ac := range acs.Items {
			//TODO(simonpasquier): provide workload bindings when implemented.
			collectBindingConditions(w, fmt.Sprintf("AlertmanagerConfig=%s/%s", ac.Namespace, ac.Name), nil)
		}
	}
}

func (ctx *TestCtx) collectStatefulSets(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		sts, err := f.KubeClient.AppsV1().StatefulSets(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get statefulsets: %v\n", ns, err)
			continue
		}

		writeYAML(w, sts)
	}
}

func (ctx *TestCtx) collectDaemonSets(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		dss, err := f.KubeClient.AppsV1().DaemonSets(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get daemonsets: %v\n", ns, err)
			continue
		}

		writeYAML(w, dss)
	}
}

func (ctx *TestCtx) collectPods(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		pods, err := f.KubeClient.CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get pods: %v\n", ns, err)
			continue
		}

		writeYAML(w, pods)
	}
}

func (ctx *TestCtx) collectServices(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		svcs, err := f.KubeClient.CoreV1().Services(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get services: %v\n", ns, err)
			continue
		}

		writeYAML(w, svcs)
	}
}

func (ctx *TestCtx) collectConfigMaps(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		cms, err := f.KubeClient.CoreV1().ConfigMaps(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get configmaps: %v\n", ns, err)
			continue
		}

		writeYAML(w, cms)
	}
}

func (ctx *TestCtx) collectSecrets(w io.Writer, f *Framework, verbose bool) {
	if !verbose {
		return
	}

	for _, ns := range ctx.namespaces {
		secs, err := f.KubeClient.CoreV1().Secrets(ns).List(context.Background(), metav1.ListOptions{})
		if err != nil {
			fmt.Fprintf(w, "%s: failed to get secrets: %v\n", ns, err)
			continue
		}

		for _, sec := range secs.Items {
			for k := range sec.Data {
				sec.Data[k] = []byte(`obfuscated`)
			}
		}

		writeYAML(w, secs)
	}
}

// ID returns an ascending ID based on the length of cleanUpFns. It is
// based on the premise that every new object also appends a new finalizerFn on
// cleanUpFns. This can e.g. be used to create multiple namespaces in the same
// test context.
func (ctx *TestCtx) ID() string {
	return ctx.id + "-" + strconv.Itoa(len(ctx.cleanUpFns))
}

func (ctx *TestCtx) Cleanup(t *testing.T) {
	t.Helper()
	var eg errgroup.Group

	for i := len(ctx.cleanUpFns) - 1; i >= 0; i-- {
		eg.Go(ctx.cleanUpFns[i])
	}

	if err := eg.Wait(); err != nil {
		t.Fatal(err)
	}
}

func (ctx *TestCtx) AddFinalizerFn(fn FinalizerFn) {
	ctx.cleanUpFns = append(ctx.cleanUpFns, fn)
}