// Copyright 2016 The prometheus-operator Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package framework import ( "context" "encoding/json" "errors" "fmt" "net/http" "slices" "strings" "testing" "time" "github.com/blang/semver/v4" "github.com/cespare/xxhash/v2" "github.com/gogo/protobuf/proto" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" certutil "k8s.io/client-go/util/cert" "k8s.io/utils/ptr" "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" v1monitoringclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" v1alpha1monitoringclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1alpha1" v1beta1monitoringclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1beta1" "github.com/prometheus-operator/prometheus-operator/pkg/operator" ) const ( prometheusOperatorServiceDeploymentName = "prometheus-operator" prometheusOperatorCertsSecretName = "prometheus-operator-certs" admissionWebhookServiceName = "prometheus-operator-admission-webhook" standaloneAdmissionHookSecretName = "admission-webhook-certs" operatorTLSDir = "/etc/tls/private" ) type Framework struct { KubeClient kubernetes.Interface MonClientV1 v1monitoringclient.MonitoringV1Interface MonClientV1alpha1 v1alpha1monitoringclient.MonitoringV1alpha1Interface MonClientV1beta1 v1beta1monitoringclient.MonitoringV1beta1Interface APIServerClient apiclient.Interface HTTPClient *http.Client MasterHost string DefaultTimeout time.Duration RestConfig *rest.Config operatorVersion semver.Version opImage string exampleDir string resourcesDir string } // New setups a test framework and returns it. func New(kubeconfig, opImage, exampleDir, resourcesDir string, operatorVersion semver.Version) (*Framework, error) { config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { return nil, fmt.Errorf("build config from flags failed: %w", err) } cli, err := kubernetes.NewForConfig(config) if err != nil { return nil, fmt.Errorf("creating new kube-client failed: %w", err) } apiCli, err := apiclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("creating new kube-client failed: %w", err) } httpc := cli.CoreV1().RESTClient().(*rest.RESTClient).Client mClientV1, err := v1monitoringclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("creating v1 monitoring client failed: %w", err) } mClientv1alpha1, err := v1alpha1monitoringclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("creating v1alpha1 monitoring client failed: %w", err) } mClientv1beta1, err := v1beta1monitoringclient.NewForConfig(config) if err != nil { return nil, fmt.Errorf("creating v1beta1 monitoring client failed: %w", err) } nodes, err := cli.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { return nil, fmt.Errorf("failed to list nodes: %w", err) } if len(nodes.Items) < 1 { return nil, errors.New("no nodes returned") } f := &Framework{ RestConfig: config, MasterHost: config.Host, KubeClient: cli, MonClientV1: mClientV1, MonClientV1alpha1: mClientv1alpha1, MonClientV1beta1: mClientv1beta1, APIServerClient: apiCli, HTTPClient: httpc, DefaultTimeout: time.Minute, operatorVersion: operatorVersion, opImage: opImage, exampleDir: exampleDir, resourcesDir: resourcesDir, } return f, nil } func (f *Framework) MakeEchoService(name, group string, serviceType v1.ServiceType) *v1.Service { service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("echo-%s", name), Labels: map[string]string{ "group": group, }, }, Spec: v1.ServiceSpec{ Type: serviceType, Ports: []v1.ServicePort{ { Name: "web", Port: 9090, TargetPort: intstr.FromString("web"), }, }, Selector: map[string]string{ "echo": name, }, }, } return service } func (f *Framework) MakeEchoDeployment(group string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "echoserver", }, Spec: appsv1.DeploymentSpec{ Replicas: proto.Int32(1), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "echo": group, }, }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ "echo": group, }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name: "echoserver", Image: "k8s.gcr.io/echoserver:1.10", Ports: []v1.ContainerPort{ { Name: "web", ContainerPort: 8443, }, }, }, }, }, }, }, } } type PrometheusOperatorOpts struct { Namespace string AllowedNamespaces []string DeniedNamespaces []string PrometheusNamespaces []string AlertmanagerNamespaces []string EnableAdmissionWebhook bool ClusterRoleBindings bool EnableScrapeConfigs bool AdditionalArgs []string EnabledFeatureGates []operator.FeatureGateName } func (f *Framework) CreateOrUpdatePrometheusOperator( ctx context.Context, namespace string, namespaceAllowlist, namespaceDenylist, prometheusInstanceNamespaces, alertmanagerInstanceNamespaces []string, createResourceAdmissionHooks, createClusterRoleBindings, createScrapeConfigCrd bool, enabledFeatureGates ...operator.FeatureGateName, ) ([]FinalizerFn, error) { return f.CreateOrUpdatePrometheusOperatorWithOpts( ctx, PrometheusOperatorOpts{ Namespace: namespace, AllowedNamespaces: namespaceAllowlist, DeniedNamespaces: namespaceDenylist, PrometheusNamespaces: prometheusInstanceNamespaces, AlertmanagerNamespaces: alertmanagerInstanceNamespaces, EnableAdmissionWebhook: createResourceAdmissionHooks, ClusterRoleBindings: createClusterRoleBindings, EnableScrapeConfigs: createScrapeConfigCrd, EnabledFeatureGates: enabledFeatureGates, }, ) } // CreateOrUpdatePrometheusOperatorWithOpts creates or updates a Prometheus // Operator Kubernetes Deployment inside the specified namespace using the // specified operator image. Semver is used to control the installation for // different versions of Prometheus Operator. In addition one can specify the // namespaces to watch, which defaults to all namespaces. It returns a slice // of functions to tear down the deployment. func (f *Framework) CreateOrUpdatePrometheusOperatorWithOpts( ctx context.Context, opts PrometheusOperatorOpts, ) ([]FinalizerFn, error) { var finalizers []FinalizerFn _, err := f.createOrUpdateServiceAccount( ctx, opts.Namespace, fmt.Sprintf("%s/rbac/prometheus-operator/prometheus-operator-service-account.yaml", f.exampleDir), ) if err != nil { return nil, fmt.Errorf("failed to create or update prometheus operator service account: %w", err) } clusterRole, err := clusterRoleFromYaml(opts.Namespace, f.exampleDir+"/rbac/prometheus-operator/prometheus-operator-cluster-role.yaml") if err != nil { return nil, fmt.Errorf("failed to load prometheus-operator cluster role: %w", err) } // Use a unique cluster role name to avoid parallel tests doing concurrent // updates to the same resource. xxh := xxhash.New() if _, err := xxh.Write([]byte(opts.Namespace)); err != nil { return nil, fmt.Errorf("failed to write hash: %w", err) } clusterRole.Name = fmt.Sprintf("%s-%x", clusterRole.Name, xxh.Sum64()) clusterRole.Rules = append(clusterRole.Rules, CRDCreateRule, CRDMonitoringRule) if slices.Contains(opts.EnabledFeatureGates, operator.PrometheusAgentDaemonSetFeature) { daemonsetRule := rbacv1.PolicyRule{ APIGroups: []string{"apps"}, Resources: []string{"daemonsets"}, Verbs: []string{"*"}, } clusterRole.Rules = append(clusterRole.Rules, daemonsetRule) } clusterRole, err = f.CreateOrUpdateClusterRole(ctx, clusterRole) if err != nil { return nil, fmt.Errorf("failed to create/update prometheus cluster role: %w", err) } finalizers = append(finalizers, func() error { return f.DeleteClusterRole(ctx, clusterRole.Name) }) if opts.ClusterRoleBindings { // Grant permissions on all namespaces. fn, err := f.createOrUpdateClusterRoleBinding(ctx, opts.Namespace, clusterRole, f.exampleDir+"/rbac/prometheus-operator/prometheus-operator-cluster-role-binding.yaml") if err != nil { return nil, fmt.Errorf("failed to create or update prometheus cluster role binding: %w", err) } finalizers = append(finalizers, fn) } else { // Grant permissions on specific namespaces. var namespaces []string namespaces = append(namespaces, opts.AllowedNamespaces...) namespaces = append(namespaces, opts.PrometheusNamespaces...) namespaces = append(namespaces, opts.AlertmanagerNamespaces...) for _, n := range namespaces { if _, err := f.createOrUpdateRoleBindingForSubjectNamespace(ctx, n, opts.Namespace, clusterRole, fmt.Sprintf("%s/prometheus-operator-role-binding.yaml", f.resourcesDir)); err != nil { return nil, fmt.Errorf("failed to create or update prometheus operator role binding: %w", err) } } } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.AlertmanagerName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.Alertmanagers(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize Alertmanager CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.PodMonitorName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.PodMonitors(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize PodMonitor CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.ProbeName, func(opts metav1.ListOptions) (object runtime.Object, err error) { return f.MonClientV1.Probes(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize Probe CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.PrometheusName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.Prometheuses(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize Prometheus CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.PrometheusRuleName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.PrometheusRules(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize PrometheusRule CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.ServiceMonitorName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.ServiceMonitors(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize ServiceMonitor CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1.ThanosRulerName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1.ThanosRulers(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize ThanosRuler CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1alpha1.AlertmanagerConfigName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1alpha1.AlertmanagerConfigs(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize AlertmanagerConfig v1alpha1 CRD: %w", err) } err = WaitForCRDReady(func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1beta1.AlertmanagerConfigs(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("wait for AlertmanagerConfig v1beta1 CRD: %w", err) } err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1alpha1.PrometheusAgentName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1alpha1.PrometheusAgents(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize PrometheusAgent v1alpha1 CRD: %w", err) } if opts.EnableScrapeConfigs { err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1alpha1.ScrapeConfigName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1alpha1.ScrapeConfigs(v1.NamespaceAll).List(ctx, opts) }) if err != nil { return nil, fmt.Errorf("initialize ScrapeConfig v1alpha1 CRD: %w", err) } } certBytes, keyBytes, err := certutil.GenerateSelfSignedCertKey(fmt.Sprintf("%s.%s.svc", prometheusOperatorServiceDeploymentName, opts.Namespace), nil, nil) if err != nil { return nil, fmt.Errorf("failed to generate certificate and key: %w", err) } if err := f.CreateOrUpdateSecretWithCert(ctx, certBytes, keyBytes, opts.Namespace, prometheusOperatorCertsSecretName); err != nil { return nil, fmt.Errorf("failed to create or update prometheus-operator TLS secret: %w", err) } deploy, err := MakeDeployment(f.exampleDir + "/rbac/prometheus-operator/prometheus-operator-deployment.yaml") if err != nil { return nil, err } // Make sure only that only one instance of the Prometheus operator is running during update. deploy.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType deploy.Spec.Template.Spec.Containers[0].Args = append(deploy.Spec.Template.Spec.Containers[0].Args, "--log-level=debug") var featureGates string if len(opts.EnabledFeatureGates) > 0 { featureGates = "-feature-gates=" } for _, fGate := range opts.EnabledFeatureGates { featureGates += fmt.Sprintf("%s=true,", fGate) } if featureGates != "" { // Remove the trailing comma deploy.Spec.Template.Spec.Containers[0].Args = append(deploy.Spec.Template.Spec.Containers[0].Args, featureGates[:len(featureGates)-1]) } var webhookServerImage string if f.opImage != "" { // Override operator image used, if specified when running tests. deploy.Spec.Template.Spec.Containers[0].Image = f.opImage repoAndTag := strings.Split(f.opImage, ":") if len(repoAndTag) != 2 { return nil, fmt.Errorf( "expected operator image '%v' split by colon to result in two substrings but got '%v'", f.opImage, repoAndTag, ) } // Override Prometheus config reloader image for i, arg := range deploy.Spec.Template.Spec.Containers[0].Args { if strings.Contains(arg, "--prometheus-config-reloader=") { deploy.Spec.Template.Spec.Containers[0].Args[i] = "--prometheus-config-reloader=" + "quay.io/prometheus-operator/prometheus-config-reloader:" + repoAndTag[1] } } webhookServerImage = "quay.io/prometheus-operator/admission-webhook:" + repoAndTag[1] } deploy.Name = prometheusOperatorServiceDeploymentName for _, ns := range opts.AllowedNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--namespaces=%v", ns), ) } for _, ns := range opts.DeniedNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--deny-namespaces=%v", ns), ) } for _, ns := range opts.PrometheusNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--prometheus-instance-namespaces=%v", ns), ) } for _, ns := range opts.AlertmanagerNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--alertmanager-instance-namespaces=%v", ns), ) } // Load the certificate and key from the created secret into the operator deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, v1.Volume{ Name: "cert", VolumeSource: v1.VolumeSource{Secret: &v1.SecretVolumeSource{SecretName: prometheusOperatorCertsSecretName}}}) deploy.Spec.Template.Spec.Containers[0].VolumeMounts = append(deploy.Spec.Template.Spec.Containers[0].VolumeMounts, v1.VolumeMount{Name: "cert", MountPath: operatorTLSDir, ReadOnly: true}) // The addition of rule admission webhooks requires TLS, so enable it and // switch to a more common https port if opts.EnableAdmissionWebhook { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, "--web.enable-tls=true", fmt.Sprintf("--web.listen-address=%v", ":8443"), ) } deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, opts.AdditionalArgs..., ) err = f.CreateOrUpdateDeploymentAndWaitUntilReady(ctx, opts.Namespace, deploy) if err != nil { return nil, err } service, err := MakeService(fmt.Sprintf("%s/rbac/prometheus-operator/prometheus-operator-service.yaml", f.exampleDir)) if err != nil { return finalizers, fmt.Errorf("cannot parse service file: %w", err) } service.Namespace = opts.Namespace service.Spec.ClusterIP = "" service.Spec.Ports = []v1.ServicePort{{Name: "https", Port: 443, TargetPort: intstr.FromInt(8443)}} if _, err := f.CreateOrUpdateServiceAndWaitUntilReady(ctx, opts.Namespace, service); err != nil { return finalizers, fmt.Errorf("failed to create or update prometheus operator service: %w", err) } if opts.EnableAdmissionWebhook { webhookService, b, err := f.CreateOrUpdateAdmissionWebhookServer(ctx, opts.Namespace, webhookServerImage) if err != nil { return nil, fmt.Errorf("failed to create webhook server: %w", err) } finalizer, err := f.createOrUpdateMutatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/prometheus-operator-mutatingwebhook.yaml", f.resourcesDir)) if err != nil { return nil, fmt.Errorf("failed to create or update mutating webhook for PrometheusRule objects: %w", err) } finalizers = append(finalizers, finalizer) finalizer, err = f.createOrUpdateValidatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/prometheus-operator-validatingwebhook.yaml", f.resourcesDir)) if err != nil { return nil, fmt.Errorf("failed to create or update validating webhook for PrometheusRule objects: %w", err) } finalizers = append(finalizers, finalizer) finalizer, err = f.createOrUpdateValidatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/alertmanager-config-validating-webhook.yaml", f.resourcesDir)) if err != nil { return nil, fmt.Errorf("failed to create or update validating webhook for AlertManagerConfig objects: %w", err) } finalizers = append(finalizers, finalizer) finalizer, err = f.configureAlertmanagerConfigConversion(ctx, webhookService, b) if err != nil { return nil, fmt.Errorf("failed to configure conversion webhook for AlertManagerConfig objects: %w", err) } finalizers = append(finalizers, finalizer) } return finalizers, nil } // DeletePrometheusOperatorClusterResource delete Prometheus Operator cluster wide resources. func (f *Framework) DeletePrometheusOperatorClusterResource(ctx context.Context) error { group := monitoring.GroupName alertmanagerCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.AlertmanagerName)) if err != nil { return fmt.Errorf("failed to make alertmanager CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", alertmanagerCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete alertmanager CRD: %w", err) } podMonitorCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.PodMonitorName)) if err != nil { return fmt.Errorf("failed to make podMonitor CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", podMonitorCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete podMonitor CRD: %w", err) } probeCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.ProbeName)) if err != nil { return fmt.Errorf("failed to make probe CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", probeCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete probe CRD: %w", err) } prometheusCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.PrometheusName)) if err != nil { return fmt.Errorf("failed to make prometheus CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", prometheusCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete prometheus CRD: %w", err) } prometheusRuleCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.PrometheusRuleName)) if err != nil { return fmt.Errorf("failed to make prometheusRule CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", prometheusRuleCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete prometheusRule CRD: %w", err) } serviceMonitorCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.ServiceMonitorName)) if err != nil { return fmt.Errorf("failed to make serviceMonitor CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", serviceMonitorCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete serviceMonitor CRD: %w", err) } thanosRulerCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1.ThanosRulerName)) if err != nil { return fmt.Errorf("failed to make thanosRuler CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", thanosRulerCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete thanosRuler CRD: %w", err) } alertmanagerConfigCRD, err := f.MakeCRD(fmt.Sprintf("%s/prometheus-operator-crd/%s_%s.yaml", f.exampleDir, group, monitoringv1alpha1.AlertmanagerConfigName)) if err != nil { return fmt.Errorf("failed to make alertmanagerConfig CRD: %w", err) } err = f.DeleteCRD(ctx, fmt.Sprintf("%s.%s", alertmanagerConfigCRD.Name, group)) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete alertmanagerConfig CRD: %w", err) } operatorMutatingHook, err := parseMutatingHookYaml(fmt.Sprintf("%s/prometheus-operator-mutatingwebhook.yaml", f.resourcesDir)) if err != nil { return fmt.Errorf("failed to parse operator mutatingwebhook: %w", err) } err = f.deleteMutatingWebhook(ctx, operatorMutatingHook.Name) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete operator mutatingwebhook: %w", err) } operatorValidatingHook, err := parseValidatingHookYaml(fmt.Sprintf("%s/prometheus-operator-validatingwebhook.yaml", f.resourcesDir)) if err != nil { return fmt.Errorf("failed to parse operator validatingwebhook: %w", err) } err = f.deleteValidatingWebhook(ctx, operatorValidatingHook.Name) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete operator mutatingwebhook: %w", err) } AlertmanagerConfigValidatingHook, err := parseValidatingHookYaml(fmt.Sprintf("%s/alertmanager-config-validating-webhook.yaml", f.resourcesDir)) if err != nil { return fmt.Errorf("failed to parse alertmanager config mutatingwebhook: %w", err) } err = f.deleteValidatingWebhook(ctx, AlertmanagerConfigValidatingHook.Name) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete alertmanager config mutatingwebhook: %w", err) } return nil } func (f *Framework) SetupPrometheusRBAC(ctx context.Context, t *testing.T, testCtx *TestCtx, ns string) { t.Helper() clusterRole, err := clusterRoleFromYaml(ns, f.exampleDir+"/rbac/prometheus/prometheus-cluster-role.yaml") if err != nil { t.Fatalf("failed to load prometheus cluster role: %v", err) } cr, err := f.CreateOrUpdateClusterRole(ctx, clusterRole) if err != nil { t.Fatalf("failed to create or update prometheus cluster role: %v", err) } finalizerFn, err := f.createOrUpdateServiceAccount(ctx, ns, f.exampleDir+"/rbac/prometheus/prometheus-service-account.yaml") if err != nil { t.Fatalf("failed to create or update prometheus service account: %v", err) } testCtx.AddFinalizerFn(finalizerFn) finalizerFn, err = f.createOrUpdateRoleBinding(ctx, ns, cr, f.resourcesDir+"/prometheus-role-binding.yml") if err != nil { t.Fatalf("failed to create prometheus role binding: %v", err) } testCtx.AddFinalizerFn(finalizerFn) } func (f *Framework) SetupPrometheusRBACGlobal(ctx context.Context, t *testing.T, testCtx *TestCtx, ns string) { t.Helper() clusterRole, err := clusterRoleFromYaml(ns, f.exampleDir+"/rbac/prometheus/prometheus-cluster-role.yaml") if err != nil { t.Fatalf("failed to load prometheus cluster role: %v", err) } if _, err := f.CreateOrUpdateClusterRole(ctx, clusterRole); err != nil && !apierrors.IsAlreadyExists(err) { t.Fatalf("failed to create or update prometheus cluster role: %v", err) } finalizerFn, err := f.createOrUpdateServiceAccount(ctx, ns, f.exampleDir+"/rbac/prometheus/prometheus-service-account.yaml") if err != nil { t.Fatalf("failed to create or update prometheus service account: %v", err) } testCtx.AddFinalizerFn(finalizerFn) finalizerFn, err = f.createOrUpdateClusterRoleBinding(ctx, ns, clusterRole, f.exampleDir+"/rbac/prometheus/prometheus-cluster-role-binding.yaml") if err != nil { t.Fatalf("failed to create or update prometheus cluster role binding: %v", err) } testCtx.AddFinalizerFn(finalizerFn) } func (f *Framework) configureAlertmanagerConfigConversion(ctx context.Context, svc *v1.Service, cert []byte) (FinalizerFn, error) { patch, err := f.MakeCRD(fmt.Sprintf("%s/alertmanager-crd-conversion/patch.json", f.exampleDir)) if err != nil { return nil, err } crd, err := f.GetCRD(ctx, patch.Name) if err != nil { return nil, err } originalBytes, err := json.Marshal(crd) if err != nil { return nil, err } patch.Spec.Conversion.Webhook.ClientConfig.Service.Name = svc.Name patch.Spec.Conversion.Webhook.ClientConfig.Service.Namespace = svc.Namespace patch.Spec.Conversion.Webhook.ClientConfig.Service.Port = &svc.Spec.Ports[0].Port patch.Spec.Conversion.Webhook.ClientConfig.CABundle = cert crd.Spec.Conversion = patch.Spec.Conversion patchBytes, err := json.Marshal(crd) if err != nil { return nil, err } jsonResult, err := strategicpatch.StrategicMergePatch(originalBytes, patchBytes, apiextensionsv1.CustomResourceDefinition{}) if err != nil { return nil, fmt.Errorf("failed to generate merge patch: %w", err) } crd, err = f.APIServerClient.ApiextensionsV1().CustomResourceDefinitions().Patch( ctx, crd.Name, types.StrategicMergePatchType, jsonResult, metav1.PatchOptions{}, ) if err != nil { return nil, fmt.Errorf("failed to patch CustomResourceDefinition object: %w", err) } if crd.Spec.Conversion.Strategy != apiextensionsv1.WebhookConverter { return nil, fmt.Errorf("expected conversion strategy to be %s, got %s", apiextensionsv1.WebhookConverter, crd.Spec.Conversion.Strategy) } finalizerFn := func() error { crd, err := f.GetCRD(ctx, patch.Name) if err != nil { return err } crd.Spec.Conversion = nil _, err = f.APIServerClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("unable to reset conversion configuration of AlertmanagerConfig CRD: %w", err) } return err } return finalizerFn, nil } // CreateOrUpdateAdmissionWebhookServer deploys an HTTPS server which acts as a // validating and mutating webhook server for PrometheusRule and // AlertManagerConfig. It is also able to convert AlertmanagerConfig objects // from v1alpha1 to v1beta1. // Returns the service and the certificate authority which can be used to trust the TLS certificate of the server. func (f *Framework) CreateOrUpdateAdmissionWebhookServer( ctx context.Context, namespace string, image string, ) (*v1.Service, []byte, error) { certBytes, keyBytes, err := certutil.GenerateSelfSignedCertKey( fmt.Sprintf("%s.%s.svc", admissionWebhookServiceName, namespace), nil, nil, ) if err != nil { return nil, nil, fmt.Errorf("failed to generate certificate and key: %w", err) } if err := f.CreateOrUpdateSecretWithCert(ctx, certBytes, keyBytes, namespace, standaloneAdmissionHookSecretName); err != nil { return nil, nil, fmt.Errorf("failed to create or update admission webhook secret: %w", err) } deploy, err := MakeDeployment(fmt.Sprintf("%s/admission-webhook/deployment.yaml", f.exampleDir)) if err != nil { return nil, nil, err } // Adjust replica count in case of single-node clusters because the // deployment manifest has anti-affinity rules. nodes, err := f.Nodes(ctx) if err != nil { return nil, nil, err } if len(nodes) == 1 { deploy.Spec.Replicas = ptr.To(int32(1)) deploy.Spec.Template.Spec.Affinity = nil deploy.Spec.Strategy = appsv1.DeploymentStrategy{} } deploy.Spec.Template.Spec.Containers[0].Args = append(deploy.Spec.Template.Spec.Containers[0].Args, "--log-level=debug") if image != "" { // Override operator image used, if specified when running tests. deploy.Spec.Template.Spec.Containers[0].Image = image repoAndTag := strings.Split(image, ":") if len(repoAndTag) != 2 { return nil, nil, fmt.Errorf( "expected image '%v' split by colon to result in two substrings but got '%v'", image, repoAndTag, ) } } _, err = f.createOrUpdateServiceAccount(ctx, namespace, fmt.Sprintf("%s/admission-webhook/service-account.yaml", f.exampleDir)) if err != nil { return nil, nil, err } err = f.CreateOrUpdateDeploymentAndWaitUntilReady(ctx, namespace, deploy) if err != nil { return nil, nil, err } service, err := MakeService(fmt.Sprintf("%s/admission-webhook/service.yaml", f.exampleDir)) if err != nil { return nil, nil, fmt.Errorf("cannot parse service file: %w", err) } service.Namespace = namespace if _, err := f.CreateOrUpdateServiceAndWaitUntilReady(ctx, namespace, service); err != nil { return nil, nil, fmt.Errorf("failed to create or update admission webhook server service: %w", err) } return service, certBytes, nil } func removeLabelsPatch(labels ...string) ([]byte, error) { type patch struct { Op string `json:"op"` Path string `json:"path"` } var patches []patch encoder := strings.NewReplacer("/", "~1", "~", "~0") for _, label := range labels { patches = append(patches, patch{Op: "remove", Path: "/metadata/labels/" + encoder.Replace(label)}) } return json.Marshal(patches) }