coreos/prometheus-operator
1
0
Fork 0
mirror of https://github.com/coreos/prometheus-operator.git synced 2026-02-05 15:46:31 +01:00
Code Issues Releases Wiki Activity

prometheus: Enqueue both P in NS as well as P selecting obj in NS

Browse Source 
By default if a new object (servicemonitor, secret, ...) is created /
updated / deleted, we reconcile all Prometheus instances in that
namespace. In addition we also need to reconcile all Prometheus
instances selecting objects (ServiceMonitors, PrometheusRules) in that
namespace.
This commit is contained in:
Max Leonard Inden
2018-07-31 17:26:26 +02:00
parent 81dc703284
commit fbf5df64ec
5 changed files with 88 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml
View File

@@ -4792,7 +4792,7 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "100 - (avg by (cpu) (irate(node_cpu{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[5m])) * 100)\n",
"expr": "1 - (avg by (cpu) (irate(node_cpu{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[5m])))\n",
"format": "time_series",
"intervalFactor": 10,
"legendFormat": "{{cpu}}",
@@ -4822,18 +4822,18 @@ items:
},
"yaxes": [
{
"format": "percent",
"format": "percentunit",
"label": null,
"logBase": 1,
"max": 100,
"max": 1,
"min": 0,
"show": true
},
{
"format": "percent",
"format": "percentunit",
"label": null,
"logBase": 1,
"max": 100,
"max": 1,
"min": 0,
"show": true
}
@@ -4883,21 +4883,21 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"} * 100",
"expr": "max(node_load1{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "load 1m",
"refId": "A"
},
{
"expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"} * 100",
"expr": "max(node_load5{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "load 5m",
"refId": "B"
},
{
"expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"} * 100",
"expr": "max(node_load15{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "load 15m",
@@ -4927,7 +4927,7 @@ items:
},
"yaxes": [
{
"format": "percent",
"format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
@@ -4935,7 +4935,7 @@ items:
"show": true
},
{
"format": "percent",
"format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
@@ -5002,28 +5002,28 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n",
"expr": "max(\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n)\n",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "memory used",
"refId": "A"
},
{
"expr": "node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}",
"expr": "max(node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "memory buffers",
"refId": "B"
},
{
"expr": "node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}",
"expr": "max(node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "memory cached",
"refId": "C"
},
{
"expr": "node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}",
"expr": "max(node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "memory free",
@@ -5131,7 +5131,7 @@ items:
"tableColumn": "",
"targets": [
{
"expr": "(\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n) * 100\n /\nnode_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n",
"expr": "max(\n (\n (\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": ""
@@ -5215,21 +5215,21 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "sum by (instance) (rate(node_disk_bytes_read{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"expr": "max(rate(node_disk_bytes_read{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "read",
"refId": "A"
},
{
"expr": "sum by (instance) (rate(node_disk_bytes_written{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"expr": "max(rate(node_disk_bytes_written{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "written",
"refId": "B"
},
{
"expr": "sum by (instance) (rate(node_disk_io_time_ms{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"expr": "max(rate(node_disk_io_time_ms{job=\"node-exporter\", instance=\"$instance\"}[2m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "io time",
@@ -5414,7 +5414,7 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "rate(node_network_receive_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m])",
"expr": "max(rate(node_network_receive_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{device}}",
@@ -5505,7 +5505,7 @@ items:
"steppedLine": false,
"targets": [
{
"expr": "rate(node_network_transmit_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m])",
"expr": "max(rate(node_network_transmit_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m]))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{device}}",

2
jsonnet/prometheus-operator/prometheus-operator.libsonnet
View File

@@ -98,7 +98,7 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet';
policyRule.withResources([
'namespaces',
]) +
policyRule.withVerbs(['list', 'watch', 'get']);
policyRule.withVerbs(['list', 'watch']);
local rules = [apiExtensionsRule, monitoringRule, appsRule, coreRule, podRule, routingRule, nodeRule, namespaceRule];

66
pkg/prometheus/operator.go
View File

@@ -697,14 +697,72 @@ func (c *Operator) enqueue(obj interface{}) {
c.queue.Add(key)
}
// enqueueForNamespace enqueues all Prometheus object keys that belong to the given namespace.
func (c *Operator) enqueueForNamespace(ns string) {
cache.ListAll(c.promInf.GetStore(), labels.Everything(), func(obj interface{}) {
// enqueueForNamespace enqueues all Prometheus object keys that belong to the
// given namespace or select objects in the given namespace.
func (c *Operator) enqueueForNamespace(nsName string) {
nsObject, exists, err := c.nsInf.GetStore().GetByKey(nsName)
if err != nil {
level.Error(c.logger).Log(
"msg", "get namespace to enqueue Prometheus instances failed",
"err", err,
)
return
}
if !exists {
level.Error(c.logger).Log(
"msg", fmt.Sprintf("get namespace to enqueue Prometheus instances failed: namespace %q does not exist", nsName),
"err", err,
)
return
}
ns := nsObject.(*v1.Namespace)
err = cache.ListAll(c.promInf.GetStore(), labels.Everything(), func(obj interface{}) {
// Check for Prometheus instances in the NS
p := obj.(*monitoringv1.Prometheus)
if p.Namespace == ns {
if p.Namespace == ns.Name {
c.enqueue(p)
return
}
// Check for Prometheus instances selecting ServiceMonitors in the NS
smNSSelector, err := metav1.LabelSelectorAsSelector(p.Spec.ServiceMonitorNamespaceSelector)
if err != nil {
level.Error(c.logger).Log(
"msg", fmt.Sprintf("failed to convert ServiceMonitorNamespaceSelector of %q to selector", p.Name),
"err", err,
)
return
}
if smNSSelector.Matches(labels.Set(ns.Labels)) {
c.enqueue(p)
return
}
// Check for Prometheus instances selecting PrometheusRules in the NS
ruleNSSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleNamespaceSelector)
if err != nil {
level.Error(c.logger).Log(
"msg", fmt.Sprintf("failed to convert RuleNamespaceSelector of %q to selector", p.Name),
"err", err,
)
return
}
if ruleNSSelector.Matches(labels.Set(ns.Labels)) {
c.enqueue(p)
return
}
})
if err != nil {
level.Error(c.logger).Log(
"msg", "listing all Prometheus instances from cache failed",
"err", err,
)
}
}
// worker runs a worker thread that just dequeues items, processes them, and marks them done.

3
scripts/delete-minikube.sh
View File

@@ -11,5 +11,4 @@ set -x
export KUBECONFIG=$HOME/.kube/config
minikube version
sudo minikube stop
sudo minikube delete
sudo minikube delete

5
test/e2e/prometheus_test.go
View File

@@ -1295,7 +1295,10 @@ func TestPrometheusGetBasicAuthSecret(t *testing.T) {
}
testNamespace := ctx.CreateNamespace(t, framework.KubeClient)
testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest)
err := testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest)
if err != nil {
t.Fatal(err)
}
simple, err := testFramework.MakeDeployment("../../test/framework/ressources/basic-auth-app-deployment.yaml")
if err != nil {