diff --git a/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml b/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml index 1143970e7..e3fa61c7b 100644 --- a/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml +++ b/contrib/kube-prometheus/manifests/grafana-dashboardDefinitions.yaml @@ -4792,7 +4792,7 @@ items: "steppedLine": false, "targets": [ { - "expr": "100 - (avg by (cpu) (irate(node_cpu{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[5m])) * 100)\n", + "expr": "1 - (avg by (cpu) (irate(node_cpu{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[5m])))\n", "format": "time_series", "intervalFactor": 10, "legendFormat": "{{cpu}}", @@ -4822,18 +4822,18 @@ items: }, "yaxes": [ { - "format": "percent", + "format": "percentunit", "label": null, "logBase": 1, - "max": 100, + "max": 1, "min": 0, "show": true }, { - "format": "percent", + "format": "percentunit", "label": null, "logBase": 1, - "max": 100, + "max": 1, "min": 0, "show": true } @@ -4883,21 +4883,21 @@ items: "steppedLine": false, "targets": [ { - "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"} * 100", + "expr": "max(node_load1{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 1m", "refId": "A" }, { - "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"} * 100", + "expr": "max(node_load5{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 5m", "refId": "B" }, { - "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"} * 100", + "expr": "max(node_load15{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "load 15m", @@ -4927,7 +4927,7 @@ items: }, "yaxes": [ { - "format": "percent", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -4935,7 +4935,7 @@ items: "show": true }, { - "format": "percent", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -5002,28 +5002,28 @@ items: "steppedLine": false, "targets": [ { - "expr": "node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n", + "expr": "max(\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory used", "refId": "A" }, { - "expr": "node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "max(node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory buffers", "refId": "B" }, { - "expr": "node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "max(node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory cached", "refId": "C" }, { - "expr": "node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "max(node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "memory free", @@ -5131,7 +5131,7 @@ items: "tableColumn": "", "targets": [ { - "expr": "(\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n- node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n) * 100\n /\nnode_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n", + "expr": "max(\n (\n (\n node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers{job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached{job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal{job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "" @@ -5215,21 +5215,21 @@ items: "steppedLine": false, "targets": [ { - "expr": "sum by (instance) (rate(node_disk_bytes_read{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_bytes_read{job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "read", "refId": "A" }, { - "expr": "sum by (instance) (rate(node_disk_bytes_written{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_bytes_written{job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "written", "refId": "B" }, { - "expr": "sum by (instance) (rate(node_disk_io_time_ms{job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "expr": "max(rate(node_disk_io_time_ms{job=\"node-exporter\", instance=\"$instance\"}[2m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "io time", @@ -5414,7 +5414,7 @@ items: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m])", + "expr": "max(rate(node_network_receive_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", @@ -5505,7 +5505,7 @@ items: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m])", + "expr": "max(rate(node_network_transmit_bytes{job=\"node-exporter\", instance=\"$instance\", device!\u007e\"lo\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{device}}", diff --git a/jsonnet/prometheus-operator/prometheus-operator.libsonnet b/jsonnet/prometheus-operator/prometheus-operator.libsonnet index 273869913..7e89589d3 100644 --- a/jsonnet/prometheus-operator/prometheus-operator.libsonnet +++ b/jsonnet/prometheus-operator/prometheus-operator.libsonnet @@ -98,7 +98,7 @@ local k = import 'ksonnet/ksonnet.beta.3/k.libsonnet'; policyRule.withResources([ 'namespaces', ]) + - policyRule.withVerbs(['list', 'watch', 'get']); + policyRule.withVerbs(['list', 'watch']); local rules = [apiExtensionsRule, monitoringRule, appsRule, coreRule, podRule, routingRule, nodeRule, namespaceRule]; diff --git a/pkg/prometheus/operator.go b/pkg/prometheus/operator.go index bda3a6b45..6080a3d95 100644 --- a/pkg/prometheus/operator.go +++ b/pkg/prometheus/operator.go @@ -697,14 +697,72 @@ func (c *Operator) enqueue(obj interface{}) { c.queue.Add(key) } -// enqueueForNamespace enqueues all Prometheus object keys that belong to the given namespace. -func (c *Operator) enqueueForNamespace(ns string) { - cache.ListAll(c.promInf.GetStore(), labels.Everything(), func(obj interface{}) { +// enqueueForNamespace enqueues all Prometheus object keys that belong to the +// given namespace or select objects in the given namespace. +func (c *Operator) enqueueForNamespace(nsName string) { + nsObject, exists, err := c.nsInf.GetStore().GetByKey(nsName) + if err != nil { + level.Error(c.logger).Log( + "msg", "get namespace to enqueue Prometheus instances failed", + "err", err, + ) + return + } + + if !exists { + level.Error(c.logger).Log( + "msg", fmt.Sprintf("get namespace to enqueue Prometheus instances failed: namespace %q does not exist", nsName), + "err", err, + ) + return + } + + ns := nsObject.(*v1.Namespace) + + err = cache.ListAll(c.promInf.GetStore(), labels.Everything(), func(obj interface{}) { + // Check for Prometheus instances in the NS p := obj.(*monitoringv1.Prometheus) - if p.Namespace == ns { + if p.Namespace == ns.Name { c.enqueue(p) + return + } + + // Check for Prometheus instances selecting ServiceMonitors in the NS + smNSSelector, err := metav1.LabelSelectorAsSelector(p.Spec.ServiceMonitorNamespaceSelector) + if err != nil { + level.Error(c.logger).Log( + "msg", fmt.Sprintf("failed to convert ServiceMonitorNamespaceSelector of %q to selector", p.Name), + "err", err, + ) + return + } + + if smNSSelector.Matches(labels.Set(ns.Labels)) { + c.enqueue(p) + return + } + + // Check for Prometheus instances selecting PrometheusRules in the NS + ruleNSSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleNamespaceSelector) + if err != nil { + level.Error(c.logger).Log( + "msg", fmt.Sprintf("failed to convert RuleNamespaceSelector of %q to selector", p.Name), + "err", err, + ) + return + } + + if ruleNSSelector.Matches(labels.Set(ns.Labels)) { + c.enqueue(p) + return } }) + if err != nil { + level.Error(c.logger).Log( + "msg", "listing all Prometheus instances from cache failed", + "err", err, + ) + } } // worker runs a worker thread that just dequeues items, processes them, and marks them done. diff --git a/scripts/delete-minikube.sh b/scripts/delete-minikube.sh index 28d59b343..865f73123 100755 --- a/scripts/delete-minikube.sh +++ b/scripts/delete-minikube.sh @@ -11,5 +11,4 @@ set -x export KUBECONFIG=$HOME/.kube/config minikube version -sudo minikube stop -sudo minikube delete \ No newline at end of file +sudo minikube delete diff --git a/test/e2e/prometheus_test.go b/test/e2e/prometheus_test.go index 4bec50489..eeef956fd 100644 --- a/test/e2e/prometheus_test.go +++ b/test/e2e/prometheus_test.go @@ -1295,7 +1295,10 @@ func TestPrometheusGetBasicAuthSecret(t *testing.T) { } testNamespace := ctx.CreateNamespace(t, framework.KubeClient) - testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest) + err := testFramework.AddLabelsToNamespace(framework.KubeClient, testNamespace, maptest) + if err != nil { + t.Fatal(err) + } simple, err := testFramework.MakeDeployment("../../test/framework/ressources/basic-auth-app-deployment.yaml") if err != nil {