From 8944ea9033323e9e1d7862c96e09c9d9953eedda Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 10 Oct 2023 11:43:24 +0200 Subject: [PATCH] chore: add e2e test for config reloader resources Signed-off-by: Simon Pasquier --- test/e2e/config_reloader_test.go | 94 +++++++++++++++++++++++++++++ test/e2e/main_test.go | 11 ++-- test/framework/framework.go | 100 +++++++++++++++++++++---------- test/framework/role_binding.go | 58 ++++++------------ 4 files changed, 188 insertions(+), 75 deletions(-) create mode 100644 test/e2e/config_reloader_test.go diff --git a/test/e2e/config_reloader_test.go b/test/e2e/config_reloader_test.go new file mode 100644 index 000000000..a5a7c68eb --- /dev/null +++ b/test/e2e/config_reloader_test.go @@ -0,0 +1,94 @@ +// Copyright 2023 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorFramework "github.com/prometheus-operator/prometheus-operator/test/framework" +) + +func testConfigReloaderResources(t *testing.T) { + testCtx := framework.NewTestCtx(t) + defer testCtx.Cleanup(t) + + ns := framework.CreateNamespace(context.Background(), t, testCtx) + framework.SetupPrometheusRBACGlobal(context.Background(), t, testCtx, ns) + + // Start Prometheus operator with the default resource requirements for the + // config reloader. + _, err := framework.CreateOrUpdatePrometheusOperatorWithOpts( + context.Background(), + operatorFramework.PrometheusOperatorOpts{ + Namespace: ns, + AllowedNamespaces: []string{ns}, + }, + ) + require.NoError(t, err) + + p := framework.MakeBasicPrometheus(ns, "instance", "instance", 1) + p, err = framework.CreatePrometheusAndWaitUntilReady(context.Background(), ns, p) + require.NoError(t, err) + + // Update the Prometheus operator deployment with new resource requirements + // for the config reloader. + var ( + cpuRequest = resource.MustParse("14m") + cpuLimit = resource.MustParse("14m") + memRequest = resource.MustParse("61Mi") + memLimit = resource.MustParse("62Mi") + ) + _, err = framework.CreateOrUpdatePrometheusOperatorWithOpts( + context.Background(), + operatorFramework.PrometheusOperatorOpts{ + Namespace: ns, + AllowedNamespaces: []string{ns}, + AdditionalArgs: []string{ + "--config-reloader-cpu-limit=" + cpuLimit.String(), + "--config-reloader-cpu-request=" + cpuRequest.String(), + "--config-reloader-memory-limit=" + memLimit.String(), + "--config-reloader-memory-request=" + memRequest.String(), + }, + }, + ) + require.NoError(t, err) + + sts, err := framework.KubeClient.AppsV1().StatefulSets(p.Namespace).Get(context.Background(), "prometheus-"+p.Name, metav1.GetOptions{}) + require.NoError(t, err) + + var resources []corev1.ResourceRequirements + var containers []corev1.Container + containers = append(containers, sts.Spec.Template.Spec.Containers...) + containers = append(containers, sts.Spec.Template.Spec.InitContainers...) + for _, c := range containers { + if c.Name == "config-reloader" || c.Name == "init-config-reloader" { + resources = append(resources, c.Resources) + } + } + require.Equal(t, 2, len(resources)) + + for _, r := range resources { + require.True(t, cpuLimit.Equal(r.Limits[corev1.ResourceCPU]), "expected %s, got %s", cpuLimit, r.Limits[corev1.ResourceCPU]) + require.True(t, cpuRequest.Equal(r.Requests[corev1.ResourceCPU]), "expected %s, got %s", cpuRequest, r.Requests[corev1.ResourceCPU]) + require.True(t, memLimit.Equal(r.Limits[corev1.ResourceMemory]), "expected %s, got %s", memLimit, r.Limits[corev1.ResourceMemory]) + require.True(t, memRequest.Equal(r.Requests[corev1.ResourceMemory]), "expected %s, got %s", memRequest, r.Requests[corev1.ResourceMemory]) + } +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 0fbf09e8e..a00196c49 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -370,11 +370,12 @@ func TestDenylist(t *testing.T) { func TestPromInstanceNs(t *testing.T) { skipPrometheusTests(t) testFuncs := map[string]func(t *testing.T){ - "AllNs": testPrometheusInstanceNamespacesAllNs, - "AllowList": testPrometheusInstanceNamespacesAllowList, - "DenyList": testPrometheusInstanceNamespacesDenyList, - "NamespaceNotFound": testPrometheusInstanceNamespacesNamespaceNotFound, - "ScrapeConfigLifecycle": testScrapeConfigLifecycle, + "AllNs": testPrometheusInstanceNamespacesAllNs, + "AllowList": testPrometheusInstanceNamespacesAllowList, + "DenyList": testPrometheusInstanceNamespacesDenyList, + "NamespaceNotFound": testPrometheusInstanceNamespacesNamespaceNotFound, + "ScrapeConfigLifecycle": testScrapeConfigLifecycle, + "ConfigReloaderResources": testConfigReloaderResources, } for name, f := range testFuncs { diff --git a/test/framework/framework.go b/test/framework/framework.go index 3f446ebe9..c0df465a3 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -195,14 +195,21 @@ func (f *Framework) MakeEchoDeployment(group string) *appsv1.Deployment { } } -// CreateOrUpdatePrometheusOperator creates or updates a Prometheus Operator Kubernetes Deployment -// inside the specified namespace using the specified operator image. Semver is used -// to control the installation for different version of Prometheus Operator. In addition -// one can specify the namespaces to watch, which defaults to all namespaces. -// Returns the CA, which can bs used to access the operator over TLS +type PrometheusOperatorOpts struct { + Namespace string + AllowedNamespaces []string + DeniedNamespaces []string + PrometheusNamespaces []string + AlertmanagerNamespaces []string + EnableAdmissionWebhook bool + ClusterRoleBindings bool + EnableScrapeConfigs bool + AdditionalArgs []string +} + func (f *Framework) CreateOrUpdatePrometheusOperator( ctx context.Context, - ns string, + namespace string, namespaceAllowlist, namespaceDenylist, prometheusInstanceNamespaces, @@ -211,12 +218,37 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( createClusterRoleBindings, createScrapeConfigCrd bool, ) ([]FinalizerFn, error) { + return f.CreateOrUpdatePrometheusOperatorWithOpts( + ctx, + PrometheusOperatorOpts{ + Namespace: namespace, + AllowedNamespaces: namespaceAllowlist, + DeniedNamespaces: namespaceDenylist, + PrometheusNamespaces: prometheusInstanceNamespaces, + AlertmanagerNamespaces: alertmanagerInstanceNamespaces, + EnableAdmissionWebhook: createResourceAdmissionHooks, + ClusterRoleBindings: createClusterRoleBindings, + EnableScrapeConfigs: createScrapeConfigCrd, + }, + ) +} + +// CreateOrUpdatePrometheusOperatorWithOpts creates or updates a Prometheus +// Operator Kubernetes Deployment inside the specified namespace using the +// specified operator image. Semver is used to control the installation for +// different versions of Prometheus Operator. In addition one can specify the +// namespaces to watch, which defaults to all namespaces. It returns a slice +// of functions to tear down the deployment. +func (f *Framework) CreateOrUpdatePrometheusOperatorWithOpts( + ctx context.Context, + opts PrometheusOperatorOpts, +) ([]FinalizerFn, error) { var finalizers []FinalizerFn _, err := f.createOrUpdateServiceAccount( ctx, - ns, + opts.Namespace, fmt.Sprintf("%s/rbac/prometheus-operator/prometheus-operator-service-account.yaml", f.exampleDir), ) if err != nil { @@ -234,17 +266,20 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( return nil, errors.Wrap(err, "failed to update prometheus cluster role") } - if createClusterRoleBindings { - if _, err := f.createOrUpdateClusterRoleBinding(ctx, ns, fmt.Sprintf("%s/rbac/prometheus-operator/prometheus-operator-cluster-role-binding.yaml", f.exampleDir)); err != nil { + if opts.ClusterRoleBindings { + // Grant permissions on all namespaces. + if _, err := f.createOrUpdateClusterRoleBinding(ctx, opts.Namespace, fmt.Sprintf("%s/rbac/prometheus-operator/prometheus-operator-cluster-role-binding.yaml", f.exampleDir)); err != nil { return nil, errors.Wrap(err, "failed to create or update prometheus cluster role binding") } } else { - namespaces := namespaceAllowlist - namespaces = append(namespaces, prometheusInstanceNamespaces...) - namespaces = append(namespaces, alertmanagerInstanceNamespaces...) + // Grant permissions on specific namespaces. + var namespaces []string + namespaces = append(namespaces, opts.AllowedNamespaces...) + namespaces = append(namespaces, opts.PrometheusNamespaces...) + namespaces = append(namespaces, opts.AlertmanagerNamespaces...) for _, n := range namespaces { - if _, err := f.CreateOrUpdateRoleBindingForSubjectNamespace(ctx, n, ns, fmt.Sprintf("%s/prometheus-operator-role-binding.yaml", f.resourcesDir)); err != nil { + if _, err := f.CreateOrUpdateRoleBindingForSubjectNamespace(ctx, n, opts.Namespace, fmt.Sprintf("%s/prometheus-operator-role-binding.yaml", f.resourcesDir)); err != nil { return nil, errors.Wrap(err, "failed to create or update prometheus operator role binding") } } @@ -320,7 +355,7 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( return nil, errors.Wrap(err, "initialize PrometheusAgent v1alpha1 CRD") } - if createScrapeConfigCrd { + if opts.EnableScrapeConfigs { err = f.CreateOrUpdateCRDAndWaitUntilReady(ctx, monitoringv1alpha1.ScrapeConfigName, func(opts metav1.ListOptions) (runtime.Object, error) { return f.MonClientV1alpha1.ScrapeConfigs(v1.NamespaceAll).List(ctx, opts) }) @@ -329,12 +364,12 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( } } - certBytes, keyBytes, err := certutil.GenerateSelfSignedCertKey(fmt.Sprintf("%s.%s.svc", prometheusOperatorServiceDeploymentName, ns), nil, nil) + certBytes, keyBytes, err := certutil.GenerateSelfSignedCertKey(fmt.Sprintf("%s.%s.svc", prometheusOperatorServiceDeploymentName, opts.Namespace), nil, nil) if err != nil { return nil, errors.Wrap(err, "failed to generate certificate and key") } - if err := f.CreateOrUpdateSecretWithCert(ctx, certBytes, keyBytes, ns, prometheusOperatorCertsSecretName); err != nil { + if err := f.CreateOrUpdateSecretWithCert(ctx, certBytes, keyBytes, opts.Namespace, prometheusOperatorCertsSecretName); err != nil { return nil, errors.Wrap(err, "failed to create or update prometheus-operator TLS secret") } @@ -343,7 +378,7 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( return nil, err } - // Make sure only one version of prometheus operator when update + // Make sure only that only one instance of the Prometheus operator is running during update. deploy.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType deploy.Spec.Template.Spec.Containers[0].Args = append(deploy.Spec.Template.Spec.Containers[0].Args, "--log-level=debug") @@ -373,28 +408,28 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( deploy.Name = prometheusOperatorServiceDeploymentName - for _, ns := range namespaceAllowlist { + for _, ns := range opts.AllowedNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--namespaces=%v", ns), ) } - for _, ns := range namespaceDenylist { + for _, ns := range opts.DeniedNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--deny-namespaces=%v", ns), ) } - for _, ns := range prometheusInstanceNamespaces { + for _, ns := range opts.PrometheusNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--prometheus-instance-namespaces=%v", ns), ) } - for _, ns := range alertmanagerInstanceNamespaces { + for _, ns := range opts.AlertmanagerNamespaces { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, fmt.Sprintf("--alertmanager-instance-namespaces=%v", ns), @@ -412,7 +447,7 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( // The addition of rule admission webhooks requires TLS, so enable it and // switch to a more common https port - if createResourceAdmissionHooks { + if opts.EnableAdmissionWebhook { deploy.Spec.Template.Spec.Containers[0].Args = append( deploy.Spec.Template.Spec.Containers[0].Args, "--web.enable-tls=true", @@ -420,7 +455,12 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( ) } - err = f.CreateOrUpdateDeploymentAndWaitUntilReady(ctx, ns, deploy) + deploy.Spec.Template.Spec.Containers[0].Args = append( + deploy.Spec.Template.Spec.Containers[0].Args, + opts.AdditionalArgs..., + ) + + err = f.CreateOrUpdateDeploymentAndWaitUntilReady(ctx, opts.Namespace, deploy) if err != nil { return nil, err } @@ -430,33 +470,33 @@ func (f *Framework) CreateOrUpdatePrometheusOperator( return finalizers, errors.Wrap(err, "cannot parse service file") } - service.Namespace = ns + service.Namespace = opts.Namespace service.Spec.ClusterIP = "" service.Spec.Ports = []v1.ServicePort{{Name: "https", Port: 443, TargetPort: intstr.FromInt(8443)}} - if _, err := f.CreateOrUpdateServiceAndWaitUntilReady(ctx, ns, service); err != nil { + if _, err := f.CreateOrUpdateServiceAndWaitUntilReady(ctx, opts.Namespace, service); err != nil { return finalizers, errors.Wrap(err, "failed to create or update prometheus operator service") } - if createResourceAdmissionHooks { - webhookService, b, err := f.CreateOrUpdateAdmissionWebhookServer(ctx, ns, webhookServerImage) + if opts.EnableAdmissionWebhook { + webhookService, b, err := f.CreateOrUpdateAdmissionWebhookServer(ctx, opts.Namespace, webhookServerImage) if err != nil { return nil, errors.Wrap(err, "failed to create webhook server") } - finalizer, err := f.createOrUpdateMutatingHook(ctx, b, ns, fmt.Sprintf("%s/prometheus-operator-mutatingwebhook.yaml", f.resourcesDir)) + finalizer, err := f.createOrUpdateMutatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/prometheus-operator-mutatingwebhook.yaml", f.resourcesDir)) if err != nil { return nil, errors.Wrap(err, "failed to create or update mutating webhook for PrometheusRule objects") } finalizers = append(finalizers, finalizer) - finalizer, err = f.createOrUpdateValidatingHook(ctx, b, ns, fmt.Sprintf("%s/prometheus-operator-validatingwebhook.yaml", f.resourcesDir)) + finalizer, err = f.createOrUpdateValidatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/prometheus-operator-validatingwebhook.yaml", f.resourcesDir)) if err != nil { return nil, errors.Wrap(err, "failed to create or update validating webhook for PrometheusRule objects") } finalizers = append(finalizers, finalizer) - finalizer, err = f.createOrUpdateValidatingHook(ctx, b, ns, fmt.Sprintf("%s/alertmanager-config-validating-webhook.yaml", f.resourcesDir)) + finalizer, err = f.createOrUpdateValidatingHook(ctx, b, opts.Namespace, fmt.Sprintf("%s/alertmanager-config-validating-webhook.yaml", f.resourcesDir)) if err != nil { return nil, errors.Wrap(err, "failed to create or update validating webhook for AlertManagerConfig objects") } diff --git a/test/framework/role_binding.go b/test/framework/role_binding.go index 27310bd67..1468a3030 100644 --- a/test/framework/role_binding.go +++ b/test/framework/role_binding.go @@ -16,6 +16,7 @@ package framework import ( "context" + "fmt" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -24,63 +25,40 @@ import ( ) func (f *Framework) CreateOrUpdateRoleBinding(ctx context.Context, ns string, relativePath string) (FinalizerFn, error) { - finalizerFn := func() error { return f.DeleteRoleBinding(ctx, ns, relativePath) } - roleBinding, err := f.parseRoleBindingYaml(relativePath) - if err != nil { - return finalizerFn, err - } - - _, err = f.KubeClient.RbacV1().RoleBindings(ns).Get(ctx, roleBinding.Name, metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return finalizerFn, err - } - - if apierrors.IsNotFound(err) { - // RoleBinding doesn't exists -> Create - _, err = f.KubeClient.RbacV1().RoleBindings(ns).Create(ctx, roleBinding, metav1.CreateOptions{}) - if err != nil { - return finalizerFn, err - } - } else { - // RoleBinding already exists -> Update - _, err = f.KubeClient.RbacV1().RoleBindings(ns).Update(ctx, roleBinding, metav1.UpdateOptions{}) - if err != nil { - return finalizerFn, err - } - } - - return finalizerFn, err + return f.CreateOrUpdateRoleBindingForSubjectNamespace(ctx, ns, "", relativePath) } func (f *Framework) CreateOrUpdateRoleBindingForSubjectNamespace(ctx context.Context, ns, subjectNs string, source string) (FinalizerFn, error) { finalizerFn := func() error { return f.DeleteRoleBinding(ctx, ns, source) } - roleBinding, err := f.parseRoleBindingYaml(source) - for i := range roleBinding.Subjects { - roleBinding.Subjects[i].Namespace = subjectNs + roleBinding, err := f.parseRoleBindingYaml(source) + if err != nil { + return nil, fmt.Errorf("failed to parse role binding manifest: %w", err) } - if err != nil { - return finalizerFn, err + if subjectNs != "" { + for i := range roleBinding.Subjects { + roleBinding.Subjects[i].Namespace = subjectNs + } } _, err = f.KubeClient.RbacV1().RoleBindings(ns).Get(ctx, roleBinding.Name, metav1.GetOptions{}) if err != nil && !apierrors.IsNotFound(err) { - return finalizerFn, err + return nil, fmt.Errorf("failed to get role binding: %w", err) } if apierrors.IsNotFound(err) { - // RoleBinding already exists -> Update - _, err = f.KubeClient.RbacV1().RoleBindings(ns).Update(ctx, roleBinding, metav1.UpdateOptions{}) - if err != nil { - return finalizerFn, err - } - } else { - // RoleBinding doesn't exists -> Create _, err = f.KubeClient.RbacV1().RoleBindings(ns).Create(ctx, roleBinding, metav1.CreateOptions{}) if err != nil { - return finalizerFn, err + return nil, fmt.Errorf("failed to create role binding: %w", err) } + + return finalizerFn, nil + } + + _, err = f.KubeClient.RbacV1().RoleBindings(ns).Update(ctx, roleBinding, metav1.UpdateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to update role binding: %w", err) } return finalizerFn, nil