From 24ea58563194dcf2d3dee8d6325ac542d29b66e6 Mon Sep 17 00:00:00 2001 From: Ha Anh Vu <75315486+haanhvu@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:01:49 +0700 Subject: [PATCH] Refactor the common implementation code (not including tests) between Prometheus's modes (#6686) * Refactor the common implementation code (not including tests) between Prometheus's modes --- pkg/operator/image.go | 8 ++ pkg/prometheus/agent/common.go | 28 +++++++ pkg/prometheus/agent/statefulset.go | 75 ++--------------- pkg/prometheus/{statefulset.go => common.go} | 83 +++++++++++++++++++ .../{statefulset_test.go => common_test.go} | 0 pkg/prometheus/server/statefulset.go | 66 +-------------- 6 files changed, 130 insertions(+), 130 deletions(-) create mode 100644 pkg/prometheus/agent/common.go rename pkg/prometheus/{statefulset.go => common.go} (86%) rename pkg/prometheus/{statefulset_test.go => common_test.go} (100%) diff --git a/pkg/operator/image.go b/pkg/operator/image.go index faf82b2a3..734108df2 100644 --- a/pkg/operator/image.go +++ b/pkg/operator/image.go @@ -56,6 +56,14 @@ func BuildImagePath(specImage, baseImage, version, tag, sha string) (string, err return baseImage + ":" + version, nil } +// BuildImagePathForAgent builds a container image path based on +// the given parameters for Prometheus Agent. +// Return specImage if not empty. +// Otherwise a combination of baseImage and version. +func BuildImagePathForAgent(specImage, baseImage, version string) (string, error) { + return BuildImagePath(specImage, baseImage, version, "", "") +} + // StringValOrDefault returns the default val if the // given string is empty/whitespace. // Otherwise returns the value of the string.. diff --git a/pkg/prometheus/agent/common.go b/pkg/prometheus/agent/common.go new file mode 100644 index 000000000..b91326132 --- /dev/null +++ b/pkg/prometheus/agent/common.go @@ -0,0 +1,28 @@ +// Copyright 2023 The prometheus-operator Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusagent + +import ( + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus" +) + +func buildAgentArgs( + cpf monitoringv1.CommonPrometheusFields, + cg *prompkg.ConfigGenerator, +) []monitoringv1.Argument { + promArgs := prompkg.BuildCommonPrometheusArgs(cpf, cg) + return appendAgentArgs(promArgs, cg, cpf.WALCompression) +} diff --git a/pkg/prometheus/agent/statefulset.go b/pkg/prometheus/agent/statefulset.go index cd17d9be0..5c742526b 100644 --- a/pkg/prometheus/agent/statefulset.go +++ b/pkg/prometheus/agent/statefulset.go @@ -30,7 +30,6 @@ import ( "github.com/prometheus-operator/prometheus-operator/pkg/k8sutil" "github.com/prometheus-operator/prometheus-operator/pkg/operator" prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus" - "github.com/prometheus-operator/prometheus-operator/pkg/webconfig" ) const ( @@ -156,12 +155,10 @@ func makeStatefulSetSpec( ) (*appsv1.StatefulSetSpec, error) { cpf := p.GetCommonPrometheusFields() - pImagePath, err := operator.BuildImagePath( + pImagePath, err := operator.BuildImagePathForAgent( ptr.Deref(cpf.Image, ""), - operator.StringValOrDefault("", c.PrometheusDefaultBaseImage), + c.PrometheusDefaultBaseImage, operator.StringValOrDefault(cpf.Version, operator.DefaultPrometheusVersion), - "", - "", ) if err != nil { return nil, err @@ -170,35 +167,14 @@ func makeStatefulSetSpec( if !slices.Contains(cpf.EnableFeatures, "agent") { cpf.EnableFeatures = append(cpf.EnableFeatures, "agent") } - promArgs := prompkg.BuildCommonPrometheusArgs(cpf, cg) - promArgs = appendAgentArgs(promArgs, cg, cpf.WALCompression) - - var ports []v1.ContainerPort - if !cpf.ListenLocal { - ports = []v1.ContainerPort{ - { - Name: cpf.PortName, - ContainerPort: 9090, - Protocol: v1.ProtocolTCP, - }, - } - } + promArgs := buildAgentArgs(cpf, cg) volumes, promVolumeMounts, err := prompkg.BuildCommonVolumes(p, tlsSecrets) if err != nil { return nil, err } - configReloaderVolumeMounts := []v1.VolumeMount{ - { - Name: "config", - MountPath: prompkg.ConfDir, - }, - { - Name: "config-out", - MountPath: prompkg.ConfOutDir, - }, - } + configReloaderVolumeMounts := prompkg.CreateConfigReloaderVolumeMounts() var configReloaderWebConfigFile string @@ -208,20 +184,11 @@ func makeStatefulSetSpec( // HTTP and HTTPS and vice-versa. webConfigGenerator := cg.WithMinimumVersion("2.24.0") if webConfigGenerator.IsCompatible() { - var fields monitoringv1.WebConfigFileFields - if cpf.Web != nil { - fields = cpf.Web.WebConfigFileFields - } - - webConfig, err := webconfig.New(prompkg.WebConfigDir, prompkg.WebConfigSecretName(p), fields) + confArg, configVol, configMount, err := prompkg.BuildWebconfig(cpf, p) if err != nil { return nil, err } - confArg, configVol, configMount, err := webConfig.GetMountParameters() - if err != nil { - return nil, err - } promArgs = append(promArgs, confArg) volumes = append(volumes, configVol...) promVolumeMounts = append(promVolumeMounts, configMount...) @@ -236,35 +203,7 @@ func makeStatefulSetSpec( webConfigGenerator.Warn("web.config.file") } - // The /-/ready handler returns OK only after the TSDB initialization has - // completed. The WAL replay can take a significant time for large setups - // hence we enable the startup probe with a generous failure threshold (15 - // minutes) to ensure that the readiness probe only comes into effect once - // Prometheus is effectively ready. - // We don't want to use the /-/healthy handler here because it returns OK as - // soon as the web server is started (irrespective of the WAL replay). - readyProbeHandler := prompkg.ProbeHandler("/-/ready", cpf, webConfigGenerator) - startupPeriodSeconds, startupFailureThreshold := prompkg.GetStatupProbePeriodSecondsAndFailureThreshold(cpf) - startupProbe := &v1.Probe{ - ProbeHandler: readyProbeHandler, - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: startupPeriodSeconds, - FailureThreshold: startupFailureThreshold, - } - - readinessProbe := &v1.Probe{ - ProbeHandler: readyProbeHandler, - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: 5, - FailureThreshold: 3, - } - - livenessProbe := &v1.Probe{ - ProbeHandler: prompkg.ProbeHandler("/-/healthy", cpf, webConfigGenerator), - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: 5, - FailureThreshold: 6, - } + startupProbe, readinessProbe, livenessProbe := prompkg.MakeProbes(cpf, webConfigGenerator) podAnnotations, podLabels := prompkg.BuildPodMetadata(cpf, cg) // In cases where an existing selector label is modified, or a new one is added, new sts cannot match existing pods. @@ -317,7 +256,7 @@ func makeStatefulSetSpec( Name: "prometheus", Image: pImagePath, ImagePullPolicy: cpf.ImagePullPolicy, - Ports: ports, + Ports: prompkg.MakeContainerPorts(cpf), Args: containerArgs, VolumeMounts: promVolumeMounts, StartupProbe: startupProbe, diff --git a/pkg/prometheus/statefulset.go b/pkg/prometheus/common.go similarity index 86% rename from pkg/prometheus/statefulset.go rename to pkg/prometheus/common.go index b64cc1697..caa2ed977 100644 --- a/pkg/prometheus/statefulset.go +++ b/pkg/prometheus/common.go @@ -32,6 +32,7 @@ import ( monitoringv1alpha1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1alpha1" "github.com/prometheus-operator/prometheus-operator/pkg/k8sutil" "github.com/prometheus-operator/prometheus-operator/pkg/operator" + "github.com/prometheus-operator/prometheus-operator/pkg/webconfig" ) const ( @@ -509,3 +510,85 @@ func GetStatupProbePeriodSecondsAndFailureThreshold(cfp monitoringv1.CommonProme return int32(startupPeriodSeconds), int32(startupFailureThreshold) } + +func MakeContainerPorts(cpf monitoringv1.CommonPrometheusFields) []v1.ContainerPort { + if cpf.ListenLocal { + return nil + } + + return []v1.ContainerPort{ + { + Name: cpf.PortName, + ContainerPort: 9090, + Protocol: v1.ProtocolTCP, + }, + } +} + +func CreateConfigReloaderVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: "config", + MountPath: ConfDir, + }, + { + Name: "config-out", + MountPath: ConfOutDir, + }, + } +} + +func BuildWebconfig( + cpf monitoringv1.CommonPrometheusFields, + p monitoringv1.PrometheusInterface, +) (monitoringv1.Argument, []v1.Volume, []v1.VolumeMount, error) { + var fields monitoringv1.WebConfigFileFields + if cpf.Web != nil { + fields = cpf.Web.WebConfigFileFields + } + + webConfig, err := webconfig.New(WebConfigDir, WebConfigSecretName(p), fields) + if err != nil { + return monitoringv1.Argument{}, nil, nil, err + } + + return webConfig.GetMountParameters() +} + +// The /-/ready handler returns OK only after the TSDB initialization has +// completed. The WAL replay can take a significant time for large setups +// hence we enable the startup probe with a generous failure threshold (15 +// minutes) to ensure that the readiness probe only comes into effect once +// Prometheus is effectively ready. +// We don't want to use the /-/healthy handler here because it returns OK as +// soon as the web server is started (irrespective of the WAL replay). +func MakeProbes( + cpf monitoringv1.CommonPrometheusFields, + webConfigGenerator *ConfigGenerator, +) (*v1.Probe, *v1.Probe, *v1.Probe) { + readyProbeHandler := ProbeHandler("/-/ready", cpf, webConfigGenerator) + startupPeriodSeconds, startupFailureThreshold := GetStatupProbePeriodSecondsAndFailureThreshold(cpf) + + startupProbe := &v1.Probe{ + ProbeHandler: readyProbeHandler, + TimeoutSeconds: ProbeTimeoutSeconds, + PeriodSeconds: startupPeriodSeconds, + FailureThreshold: startupFailureThreshold, + } + + readinessProbe := &v1.Probe{ + ProbeHandler: readyProbeHandler, + TimeoutSeconds: ProbeTimeoutSeconds, + PeriodSeconds: 5, + FailureThreshold: 3, + } + + livenessProbe := &v1.Probe{ + ProbeHandler: ProbeHandler("/-/healthy", cpf, webConfigGenerator), + TimeoutSeconds: ProbeTimeoutSeconds, + PeriodSeconds: 5, + FailureThreshold: 6, + } + + return startupProbe, readinessProbe, livenessProbe +} diff --git a/pkg/prometheus/statefulset_test.go b/pkg/prometheus/common_test.go similarity index 100% rename from pkg/prometheus/statefulset_test.go rename to pkg/prometheus/common_test.go diff --git a/pkg/prometheus/server/statefulset.go b/pkg/prometheus/server/statefulset.go index 0aebf0d10..d6ebef65f 100644 --- a/pkg/prometheus/server/statefulset.go +++ b/pkg/prometheus/server/statefulset.go @@ -30,7 +30,6 @@ import ( "github.com/prometheus-operator/prometheus-operator/pkg/k8sutil" "github.com/prometheus-operator/prometheus-operator/pkg/operator" prompkg "github.com/prometheus-operator/prometheus-operator/pkg/prometheus" - "github.com/prometheus-operator/prometheus-operator/pkg/webconfig" ) const ( @@ -245,33 +244,13 @@ func makeStatefulSetSpec( promArgs := prompkg.BuildCommonPrometheusArgs(cpf, cg) promArgs = appendServerArgs(promArgs, cg, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, cpf.WALCompression) - var ports []v1.ContainerPort - if !cpf.ListenLocal { - ports = []v1.ContainerPort{ - { - Name: cpf.PortName, - ContainerPort: 9090, - Protocol: v1.ProtocolTCP, - }, - } - } - volumes, promVolumeMounts, err := prompkg.BuildCommonVolumes(p, tlsSecrets) if err != nil { return nil, err } volumes, promVolumeMounts = appendServerVolumes(volumes, promVolumeMounts, queryLogFile, ruleConfigMapNames) - configReloaderVolumeMounts := []v1.VolumeMount{ - { - Name: "config", - MountPath: prompkg.ConfDir, - }, - { - Name: "config-out", - MountPath: prompkg.ConfOutDir, - }, - } + configReloaderVolumeMounts := prompkg.CreateConfigReloaderVolumeMounts() var configReloaderWebConfigFile string @@ -281,20 +260,11 @@ func makeStatefulSetSpec( // HTTP and HTTPS and vice-versa. webConfigGenerator := cg.WithMinimumVersion("2.24.0") if webConfigGenerator.IsCompatible() { - var fields monitoringv1.WebConfigFileFields - if cpf.Web != nil { - fields = cpf.Web.WebConfigFileFields - } - - webConfig, err := webconfig.New(prompkg.WebConfigDir, prompkg.WebConfigSecretName(p), fields) + confArg, configVol, configMount, err := prompkg.BuildWebconfig(cpf, p) if err != nil { return nil, err } - confArg, configVol, configMount, err := webConfig.GetMountParameters() - if err != nil { - return nil, err - } promArgs = append(promArgs, confArg) volumes = append(volumes, configVol...) promVolumeMounts = append(promVolumeMounts, configMount...) @@ -309,35 +279,7 @@ func makeStatefulSetSpec( webConfigGenerator.Warn("web.config.file") } - // The /-/ready handler returns OK only after the TSDB initialization has - // completed. The WAL replay can take a significant time for large setups - // hence we enable the startup probe with a generous failure threshold (15 - // minutes) to ensure that the readiness probe only comes into effect once - // Prometheus is effectively ready. - // We don't want to use the /-/healthy handler here because it returns OK as - // soon as the web server is started (irrespective of the WAL replay). - readyProbeHandler := prompkg.ProbeHandler("/-/ready", cpf, webConfigGenerator) - startupPeriodSeconds, startupFailureThreshold := prompkg.GetStatupProbePeriodSecondsAndFailureThreshold(cpf) - startupProbe := &v1.Probe{ - ProbeHandler: readyProbeHandler, - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: startupPeriodSeconds, - FailureThreshold: startupFailureThreshold, - } - - readinessProbe := &v1.Probe{ - ProbeHandler: readyProbeHandler, - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: 5, - FailureThreshold: 3, - } - - livenessProbe := &v1.Probe{ - ProbeHandler: prompkg.ProbeHandler("/-/healthy", cpf, webConfigGenerator), - TimeoutSeconds: prompkg.ProbeTimeoutSeconds, - PeriodSeconds: 5, - FailureThreshold: 6, - } + startupProbe, readinessProbe, livenessProbe := prompkg.MakeProbes(cpf, webConfigGenerator) podAnnotations, podLabels := prompkg.BuildPodMetadata(cpf, cg) // In cases where an existing selector label is modified, or a new one is added, new sts cannot match existing pods. @@ -417,7 +359,7 @@ func makeStatefulSetSpec( Name: "prometheus", Image: pImagePath, ImagePullPolicy: cpf.ImagePullPolicy, - Ports: ports, + Ports: prompkg.MakeContainerPorts(cpf), Args: containerArgs, VolumeMounts: promVolumeMounts, StartupProbe: startupProbe,