1
0
mirror of https://github.com/coreos/prometheus-operator.git synced 2026-02-05 06:45:27 +01:00
Files
prometheus-operator/test/e2e/status_subresource_test.go
Y@&h 7a2bb75866 Remove binding from PodMonitor's status when workload deletd or invalid Ref (#7936)
* feat:  PodMonitor binding removal

* feat: tests for binding removals from podMon status
2025-09-23 07:41:58 +00:00

454 lines
18 KiB
Go

// Copyright 2022 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package e2e
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
"github.com/prometheus-operator/prometheus-operator/pkg/operator"
testFramework "github.com/prometheus-operator/prometheus-operator/test/framework"
)
// testFinalizerWhenStatusForConfigResourcesEnabled tests the adding/removing of status-cleanup finalizer for Prometheus when StatusForConfigurationResourcesFeature is enabled.
func testFinalizerWhenStatusForConfigResourcesEnabled(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "status-cleanup-finalizer-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
p, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
finalizers := p.GetFinalizers()
require.NotEmpty(t, finalizers)
err = framework.DeletePrometheusAndWaitUntilGone(ctx, ns, name)
require.NoError(t, err)
}
// testServiceMonitorStatusSubresource validates ServiceMonitor status updates upon Prometheus selection.
func testServiceMonitorStatusSubresource(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "servicemonitor-status-subresource-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
// Create a first service monitor to check that the operator only updates the binding when needed.
sm1 := framework.MakeBasicServiceMonitor("smon1")
sm1.Labels["group"] = name
sm1, err = framework.MonClientV1.ServiceMonitors(ns).Create(ctx, sm1, v1.CreateOptions{})
require.NoError(t, err)
// Record the lastTransitionTime value.
sm1, err = framework.WaitForServiceMonitorCondition(ctx, sm1, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
binding, err := framework.GetWorkloadBinding(sm1.Status.Bindings, p, monitoringv1.PrometheusName)
require.NoError(t, err)
cond, err := framework.GetConfigResourceCondition(binding.Conditions, monitoringv1.Accepted)
require.NoError(t, err)
ts := cond.LastTransitionTime.String()
require.NotEqual(t, "", ts)
// Create a second service monitor to check that the operator updates the binding when the condition changes.
sm2 := framework.MakeBasicServiceMonitor("smon2")
sm2.Labels["group"] = name
sm2, err = framework.MonClientV1.ServiceMonitors(ns).Create(ctx, sm2, v1.CreateOptions{})
require.NoError(t, err)
sm2, err = framework.WaitForServiceMonitorCondition(ctx, sm2, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
// Update the labels of the first service monitor. A label update doesn't
// change the status of the service monitor and the observed timetstamp
// should be the same as before.
sm1.Labels["test"] = "test"
sm1, err = framework.MonClientV1.ServiceMonitors(ns).Update(ctx, sm1, v1.UpdateOptions{})
require.NoError(t, err)
// Update the second service monitor to reference an non-existing Secret.
sm2.Spec.Endpoints[0].BasicAuth = &monitoringv1.BasicAuth{
Username: corev1.SecretKeySelector{
Key: "username",
LocalObjectReference: corev1.LocalObjectReference{
Name: name,
},
},
}
sm2, err = framework.MonClientV1.ServiceMonitors(ns).Update(ctx, sm2, v1.UpdateOptions{})
require.NoError(t, err)
// The second ServiceMonitor should change to Accepted=False.
_, err = framework.WaitForServiceMonitorCondition(ctx, sm2, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionFalse, 1*time.Minute)
require.NoError(t, err)
// The first ServiceMonitor should remain unchanged.
sm1, err = framework.WaitForServiceMonitorCondition(ctx, sm1, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
binding, err = framework.GetWorkloadBinding(sm1.Status.Bindings, p, monitoringv1.PrometheusName)
require.NoError(t, err)
cond, err = framework.GetConfigResourceCondition(binding.Conditions, monitoringv1.Accepted)
require.NoError(t, err)
require.Equal(t, ts, cond.LastTransitionTime.String())
}
// testGarbageCollectionOfServiceMonitorBinding validates that the operator removes the reference to the Prometheus resource when the ServiceMonitor isn't selected anymore by the workload.
func testGarbageCollectionOfServiceMonitorBinding(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "smon-status-binding-cleanup-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
sm := framework.MakeBasicServiceMonitor(name)
sm, err = framework.MonClientV1.ServiceMonitors(ns).Create(ctx, sm, v1.CreateOptions{})
require.NoError(t, err)
sm, err = framework.WaitForServiceMonitorCondition(ctx, sm, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
// Update the ServiceMonitor's labels, Prometheus doesn't select the resource anymore.
sm.Labels = map[string]string{}
sm, err = framework.MonClientV1.ServiceMonitors(ns).Update(ctx, sm, v1.UpdateOptions{})
require.NoError(t, err)
_, err = framework.WaitForServiceMonitorWorkloadBindingCleanup(ctx, sm, p, monitoringv1.PrometheusName, 1*time.Minute)
require.NoError(t, err)
}
func testServiceMonitorStatusWithMultipleWorkloads(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "servicemonitor-status-multiple-workloads"
p1 := framework.MakeBasicPrometheus(ns, "server1", name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p1)
require.NoError(t, err)
p2 := framework.MakeBasicPrometheus(ns, "server2", name, 1)
// Forbid access to the container's filesystem.
p2.Spec.ArbitraryFSAccessThroughSMs.Deny = true
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p2)
require.NoError(t, err)
sm := framework.MakeBasicServiceMonitor(name)
sm.Spec.Endpoints[0].BearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
sm, err = framework.MonClientV1.ServiceMonitors(ns).Create(ctx, sm, v1.CreateOptions{})
require.NoError(t, err)
// The ServiceMonitor should be accepted by the Prometheus "server1" resource.
_, err = framework.WaitForServiceMonitorCondition(ctx, sm, p1, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
// ServiceMonitor should be rejected by the Prometheus "server2" resource because it wants to access the SA token file.
_, err = framework.WaitForServiceMonitorCondition(ctx, sm, p2, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionFalse, 1*time.Minute)
require.NoError(t, err)
}
// testRmServiceMonitorBindingDuringWorkloadDelete validates that the operator removes the reference to the Prometheus resource when workload is deleted.
func testRmServiceMonitorBindingDuringWorkloadDelete(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "workload-del-smon-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err, "failed to create Prometheus")
smon := framework.MakeBasicServiceMonitor(name)
sm, err := framework.MonClientV1.ServiceMonitors(ns).Create(ctx, smon, v1.CreateOptions{})
require.NoError(t, err)
sm, err = framework.WaitForServiceMonitorCondition(ctx, sm, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
err = framework.DeletePrometheusAndWaitUntilGone(ctx, ns, name)
require.NoError(t, err)
_, err = framework.WaitForServiceMonitorWorkloadBindingCleanup(ctx, sm, p, monitoringv1.PrometheusName, 1*time.Minute)
require.NoError(t, err)
}
// testPodMonitorStatusSubresource validates PodMonitor status updates upon Prometheus selection.
func testPodMonitorStatusSubresource(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "podmonitor-status-subresource-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
// Create a first podmonitor to check that the operator only updates the binding when needed.
pm1 := framework.MakeBasicPodMonitor("pmon1")
pm1.Labels["group"] = name
pm1, err = framework.MonClientV1.PodMonitors(ns).Create(ctx, pm1, v1.CreateOptions{})
require.NoError(t, err)
// Record the lastTransitionTime value.
pm1, err = framework.WaitForPodMonitorCondition(ctx, pm1, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
binding, err := framework.GetWorkloadBinding(pm1.Status.Bindings, p, monitoringv1.PrometheusName)
require.NoError(t, err)
cond, err := framework.GetConfigResourceCondition(binding.Conditions, monitoringv1.Accepted)
require.NoError(t, err)
ts := cond.LastTransitionTime.String()
require.NotEqual(t, "", ts)
// Create a second podmonitor to check that the operator updates the binding when the condition changes.
pm2 := framework.MakeBasicPodMonitor("pmon2")
pm2.Labels["group"] = name
pm2, err = framework.MonClientV1.PodMonitors(ns).Create(ctx, pm2, v1.CreateOptions{})
require.NoError(t, err)
pm2, err = framework.WaitForPodMonitorCondition(ctx, pm2, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
// Update the labels of the first podmonitor. A label update doesn't
// change the status of the podmonitor and the observed timetstamp
// should be the same as before.
pm1.Labels["test"] = "test"
pm1, err = framework.MonClientV1.PodMonitors(ns).Update(ctx, pm1, v1.UpdateOptions{})
require.NoError(t, err)
// Update the second podmonitor to reference an non-existing Secret.
pm2.Spec.PodMetricsEndpoints[0].BasicAuth = &monitoringv1.BasicAuth{
Username: corev1.SecretKeySelector{
Key: "username",
LocalObjectReference: corev1.LocalObjectReference{
Name: name,
},
},
}
pm2, err = framework.MonClientV1.PodMonitors(ns).Update(ctx, pm2, v1.UpdateOptions{})
require.NoError(t, err)
// The second PodMonitor should change to Accepted=False.
_, err = framework.WaitForPodMonitorCondition(ctx, pm2, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionFalse, 1*time.Minute)
require.NoError(t, err)
// The first PodMonitor should remain unchanged.
pm1, err = framework.WaitForPodMonitorCondition(ctx, pm1, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
binding, err = framework.GetWorkloadBinding(pm1.Status.Bindings, p, monitoringv1.PrometheusName)
require.NoError(t, err)
cond, err = framework.GetConfigResourceCondition(binding.Conditions, monitoringv1.Accepted)
require.NoError(t, err)
require.Equal(t, ts, cond.LastTransitionTime.String())
}
// testGarbageCollectionOfPodMonitorBinding validates that the operator removes the reference to the Prometheus resource when the PodMonitor isn't selected anymore by the workload.
func testGarbageCollectionOfPodMonitorBinding(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "pmon-status-binding-cleanup-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
pm := framework.MakeBasicPodMonitor(name)
pm, err = framework.MonClientV1.PodMonitors(ns).Create(ctx, pm, v1.CreateOptions{})
require.NoError(t, err)
pm, err = framework.WaitForPodMonitorCondition(ctx, pm, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
// Update the PodMonitor's labels, Prometheus doesn't select the resource anymore.
pm.Labels = map[string]string{}
pm, err = framework.MonClientV1.PodMonitors(ns).Update(ctx, pm, v1.UpdateOptions{})
require.NoError(t, err)
_, err = framework.WaitForPodMonitorWorkloadBindingCleanup(ctx, pm, p, monitoringv1.PrometheusName, 1*time.Minute)
require.NoError(t, err)
}
// testRmPodMonitorBindingDuringWorkloadDelete validates that the operator removes the reference to the Prometheus resource from PodMonitor's status when workload is deleted.
func testRmPodMonitorBindingDuringWorkloadDelete(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "workload-del-pmon-test"
p := framework.MakeBasicPrometheus(ns, name, name, 1)
_, err = framework.CreatePrometheusAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err, "failed to create Prometheus")
pmon := framework.MakeBasicPodMonitor(name)
pm, err := framework.MonClientV1.PodMonitors(ns).Create(ctx, pmon, v1.CreateOptions{})
require.NoError(t, err)
pm, err = framework.WaitForPodMonitorCondition(ctx, pm, p, monitoringv1.PrometheusName, monitoringv1.Accepted, monitoringv1.ConditionTrue, 1*time.Minute)
require.NoError(t, err)
err = framework.DeletePrometheusAndWaitUntilGone(ctx, ns, name)
require.NoError(t, err)
_, err = framework.WaitForPodMonitorWorkloadBindingCleanup(ctx, pm, p, monitoringv1.PrometheusName, 1*time.Minute)
require.NoError(t, err)
}
// testFinalizerForPromAgentWhenStatusForConfigResEnabled tests the adding/removing of status-cleanup finalizer for PrometheusAgent when StatusForConfigurationResourcesFeature is enabled.
func testFinalizerForPromAgentWhenStatusForConfigResEnabled(t *testing.T) {
t.Parallel()
ctx := context.Background()
testCtx := framework.NewTestCtx(t)
defer testCtx.Cleanup(t)
ns := framework.CreateNamespace(ctx, t, testCtx)
framework.SetupPrometheusRBAC(ctx, t, testCtx, ns)
_, err := framework.CreateOrUpdatePrometheusOperatorWithOpts(
ctx, testFramework.PrometheusOperatorOpts{
Namespace: ns,
AllowedNamespaces: []string{ns},
EnabledFeatureGates: []operator.FeatureGateName{operator.StatusForConfigurationResourcesFeature},
},
)
require.NoError(t, err)
name := "status-cleanup-finalizer-test"
p := framework.MakeBasicPrometheusAgent(ns, name, name, 1)
p, err = framework.CreatePrometheusAgentAndWaitUntilReady(ctx, ns, p)
require.NoError(t, err)
finalizers := p.GetFinalizers()
require.NotEmpty(t, finalizers)
err = framework.DeletePrometheusAgentAndWaitUntilGone(ctx, ns, name)
require.NoError(t, err)
}