From 29f781fdae06e90fa0328080e59a8b331374f814 Mon Sep 17 00:00:00 2001 From: Brian Burt Date: Thu, 19 May 2022 10:52:48 -0400 Subject: [PATCH] RHDEVDOCS-4007-fully-modularize-monitoring-docs --- .../monitoring-accessing-the-alerting-ui.adoc | 1 + ...ing-custom-alertmanager-configuration.adoc | 7 ---- ...ng-cluster-for-application-monitoring.adoc | 1 + ...iguring-monitoring-for-an-application.adoc | 1 + ...toring-configuring-persistent-storage.adoc | 23 +++++++++++++ ...ling-query-logging-for-thanos-querier.adoc | 2 +- .../monitoring-managing-alerting-rules.adoc | 1 + modules/monitoring-managing-silences.adoc | 1 + ...ng-alerting-for-user-defined-projects.adoc | 1 + modules/monitoring-querying-metrics.adoc | 1 + ...ng-alerts-silences-and-alerting-rules.adoc | 1 + ...arching-and-filtering-metrics-targets.adoc | 1 + ...ing-notifications-to-external-systems.adoc | 1 + ...setting-query-log-file-for-prometheus.adoc | 2 +- ...-collection-for-user-defined-projects.adoc | 1 + .../monitoring-support-considerations.adoc | 1 + modules/monitoring-understanding-metrics.adoc | 5 --- ...toring-unmanaged-monitoring-operators.adoc | 1 + .../configuring-the-monitoring-stack.adoc | 34 ++++--------------- monitoring/managing-alerts.adoc | 7 ++++ monitoring/managing-metrics.adoc | 5 +++ 21 files changed, 56 insertions(+), 42 deletions(-) create mode 100644 modules/monitoring-configuring-persistent-storage.adoc diff --git a/modules/monitoring-accessing-the-alerting-ui.adoc b/modules/monitoring-accessing-the-alerting-ui.adoc index f6bbc0a7af..f76b00f833 100644 --- a/modules/monitoring-accessing-the-alerting-ui.adoc +++ b/modules/monitoring-accessing-the-alerting-ui.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-alerts.adoc +:_content-type: PROCEDURE [id="accessing_the_alerting_ui_{context}"] = Accessing the Alerting UI in the Administrator and Developer perspectives diff --git a/modules/monitoring-applying-custom-alertmanager-configuration.adoc b/modules/monitoring-applying-custom-alertmanager-configuration.adoc index 277ffe3e6d..12b4adf1f8 100644 --- a/modules/monitoring-applying-custom-alertmanager-configuration.adoc +++ b/modules/monitoring-applying-custom-alertmanager-configuration.adoc @@ -101,10 +101,3 @@ To change the Alertmanager configuration from the {product-title} web console: . Modify the YAML configuration file. . Select *Save*. - -[role="_additional-resources"] -.Additional resources - -* See link:https://www.pagerduty.com/[the PagerDuty official site] for more information on PagerDuty -* See link:https://www.pagerduty.com/docs/guides/prometheus-integration-guide/[the PagerDuty Prometheus Integration Guide] to learn how to retrieve the `service_key` -* See link:https://prometheus.io/docs/alerting/configuration/[Alertmanager configuration] for configuring alerting through different alert receivers diff --git a/modules/monitoring-configuring-cluster-for-application-monitoring.adoc b/modules/monitoring-configuring-cluster-for-application-monitoring.adoc index 94c24d136e..6b34961719 100644 --- a/modules/monitoring-configuring-cluster-for-application-monitoring.adoc +++ b/modules/monitoring-configuring-cluster-for-application-monitoring.adoc @@ -2,6 +2,7 @@ // // * monitoring/application-monitoring.adoc +:_content-type: PROCEDURE [id="configuring-cluster-for-application-monitoring_{context}"] = Configuring cluster for application monitoring diff --git a/modules/monitoring-configuring-monitoring-for-an-application.adoc b/modules/monitoring-configuring-monitoring-for-an-application.adoc index 888ba77439..50bd5bf072 100644 --- a/modules/monitoring-configuring-monitoring-for-an-application.adoc +++ b/modules/monitoring-configuring-monitoring-for-an-application.adoc @@ -2,6 +2,7 @@ // // * monitoring/application-monitoring.adoc +:_content-type: PROCEDURE [id="configuring-monitoring-for-an-application_{context}"] = Configuring monitoring for an application diff --git a/modules/monitoring-configuring-persistent-storage.adoc b/modules/monitoring-configuring-persistent-storage.adoc new file mode 100644 index 0000000000..e410c0ace2 --- /dev/null +++ b/modules/monitoring-configuring-persistent-storage.adoc @@ -0,0 +1,23 @@ +// Module included in the following assemblies: +// +// * monitoring/configuring-the-monitoring-stack.adoc + +:_content-type: CONCEPT +[id="configuring_persistent_storage_{context}"] += Configuring persistent storage + +Running cluster monitoring with persistent storage means that your metrics are stored to a persistent volume (PV) and can survive a pod being restarted or recreated. This is ideal if you require your metrics or alerting data to be guarded from data loss. For production environments, it is highly recommended to configure persistent storage. Because of the high IO demands, it is advantageous to use local storage. + +[id="persistent-storage-prerequisites"] +== Persistent storage prerequisites + +* Dedicate sufficient local persistent storage to ensure that the disk does not become full. How much storage you need depends on the number of pods. + +* Make sure you have a persistent volume (PV) ready to be claimed by the persistent volume claim (PVC), one PV for each replica. Because Prometheus has two replicas and Alertmanager has three replicas, you need five PVs to support the entire monitoring stack. The PVs should be available from the Local Storage Operator. This does not apply if you enable dynamically provisioned storage. + +* Use the block type of storage. + +[NOTE] +==== +If you use a local volume for persistent storage, do not use a raw block volume, which is described with `volumeMode: block` in the `LocalVolume` object. Prometheus cannot use raw block volumes. +==== \ No newline at end of file diff --git a/modules/monitoring-enabling-query-logging-for-thanos-querier.adoc b/modules/monitoring-enabling-query-logging-for-thanos-querier.adoc index caa41df825..0359b93096 100644 --- a/modules/monitoring-enabling-query-logging-for-thanos-querier.adoc +++ b/modules/monitoring-enabling-query-logging-for-thanos-querier.adoc @@ -2,8 +2,8 @@ // // * monitoring/configuring-the-monitoring-stack.adoc -[id="enabling-query-logging-for-thanos-querier_{context}"] :_content-type: PROCEDURE +[id="enabling-query-logging-for-thanos-querier_{context}"] = Enabling query logging for Thanos Querier [role="_abstract"] diff --git a/modules/monitoring-managing-alerting-rules.adoc b/modules/monitoring-managing-alerting-rules.adoc index aab1dbfd4e..eda68feee6 100644 --- a/modules/monitoring-managing-alerting-rules.adoc +++ b/modules/monitoring-managing-alerting-rules.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-alerts.adoc +:_content-type: CONCEPT [id="managing-alerting-rules_{context}"] = Managing alerting rules diff --git a/modules/monitoring-managing-silences.adoc b/modules/monitoring-managing-silences.adoc index 00a4c6b0fe..3105ac0056 100644 --- a/modules/monitoring-managing-silences.adoc +++ b/modules/monitoring-managing-silences.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-alerts.adoc +:_content-type: CONCEPT [id="managing-silences_{context}"] = Managing silences diff --git a/modules/monitoring-optimizing-alerting-for-user-defined-projects.adoc b/modules/monitoring-optimizing-alerting-for-user-defined-projects.adoc index 10c57e5d9c..f4d98537bb 100644 --- a/modules/monitoring-optimizing-alerting-for-user-defined-projects.adoc +++ b/modules/monitoring-optimizing-alerting-for-user-defined-projects.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-alerts.adoc +:_content-type: CONCEPT [id="Optimizing-alerting-for-user-defined-projects_{context}"] = Optimizing alerting for user-defined projects diff --git a/modules/monitoring-querying-metrics.adoc b/modules/monitoring-querying-metrics.adoc index 43c400a18c..28e5ca8769 100644 --- a/modules/monitoring-querying-metrics.adoc +++ b/modules/monitoring-querying-metrics.adoc @@ -3,6 +3,7 @@ // * monitoring/managing-metrics.adoc // * virt/logging_events_monitoring/virt-prometheus-queries.adoc +:_content-type: CONCEPT [id="querying-metrics_{context}"] = Querying metrics diff --git a/modules/monitoring-searching-alerts-silences-and-alerting-rules.adoc b/modules/monitoring-searching-alerts-silences-and-alerting-rules.adoc index 5871bfaed5..46747994a4 100644 --- a/modules/monitoring-searching-alerts-silences-and-alerting-rules.adoc +++ b/modules/monitoring-searching-alerts-silences-and-alerting-rules.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-alerts.adoc +:_content-type: CONCEPT [id="searching-alerts-silences-and-alerting-rules_{context}"] = Searching and filtering alerts, silences, and alerting rules diff --git a/modules/monitoring-searching-and-filtering-metrics-targets.adoc b/modules/monitoring-searching-and-filtering-metrics-targets.adoc index 0a7547122e..1253dc9909 100644 --- a/modules/monitoring-searching-and-filtering-metrics-targets.adoc +++ b/modules/monitoring-searching-and-filtering-metrics-targets.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-metrics-targets.adoc +:_content-type: CONCEPT [id="monitoring-searching-and-filtering-metrics-targets_{context}"] = Searching and filtering metrics targets diff --git a/modules/monitoring-sending-notifications-to-external-systems.adoc b/modules/monitoring-sending-notifications-to-external-systems.adoc index 5b88411fa5..2eef40f01f 100644 --- a/modules/monitoring-sending-notifications-to-external-systems.adoc +++ b/modules/monitoring-sending-notifications-to-external-systems.adoc @@ -3,6 +3,7 @@ // * monitoring/managing-alerts.adoc // * post_installation_configuration/configuring-alert-notifications.adoc +:_content-type: CONCEPT [id="sending-notifications-to-external-systems_{context}"] = Sending notifications to external systems diff --git a/modules/monitoring-setting-query-log-file-for-prometheus.adoc b/modules/monitoring-setting-query-log-file-for-prometheus.adoc index 3b8bcb1fe5..ded1516498 100644 --- a/modules/monitoring-setting-query-log-file-for-prometheus.adoc +++ b/modules/monitoring-setting-query-log-file-for-prometheus.adoc @@ -2,8 +2,8 @@ // // * monitoring/configuring-the-monitoring-stack.adoc -[id="setting-query-log-file-for-prometheus_{context}"] :_content-type: PROCEDURE +[id="setting-query-log-file-for-prometheus_{context}"] = Enabling the query log file for Prometheus [role="_abstract"] diff --git a/modules/monitoring-setting-up-metrics-collection-for-user-defined-projects.adoc b/modules/monitoring-setting-up-metrics-collection-for-user-defined-projects.adoc index 92a45bacb5..29d2c1fe72 100644 --- a/modules/monitoring-setting-up-metrics-collection-for-user-defined-projects.adoc +++ b/modules/monitoring-setting-up-metrics-collection-for-user-defined-projects.adoc @@ -2,6 +2,7 @@ // // * monitoring/managing-metrics.adoc +:_content-type: CONCEPT [id="setting-up-metrics-collection-for-user-defined-projects_{context}"] = Setting up metrics collection for user-defined projects diff --git a/modules/monitoring-support-considerations.adoc b/modules/monitoring-support-considerations.adoc index 6e49bda5c7..8a1c1b7045 100644 --- a/modules/monitoring-support-considerations.adoc +++ b/modules/monitoring-support-considerations.adoc @@ -2,6 +2,7 @@ // // * monitoring/configuring-the-monitoring-stack.adoc +:_content-type: CONCEPT [id="support-considerations_{context}"] = Support considerations for monitoring diff --git a/modules/monitoring-understanding-metrics.adoc b/modules/monitoring-understanding-metrics.adoc index 6e442adb61..cbca81148a 100644 --- a/modules/monitoring-understanding-metrics.adoc +++ b/modules/monitoring-understanding-metrics.adoc @@ -29,8 +29,3 @@ http_requests_total{code="404",method="get"} 2 # TYPE version gauge version{version="v0.1.0"} 1 ---- - -[role="_additional-resources"] -.Additional resources - -* See the link:https://prometheus.io/docs/instrumenting/clientlibs/[Prometheus documentation] for details on Prometheus client libraries. diff --git a/modules/monitoring-unmanaged-monitoring-operators.adoc b/modules/monitoring-unmanaged-monitoring-operators.adoc index 6cb3251484..9d3ba3631e 100644 --- a/modules/monitoring-unmanaged-monitoring-operators.adoc +++ b/modules/monitoring-unmanaged-monitoring-operators.adoc @@ -2,6 +2,7 @@ // // * monitoring/configuring-the-monitoring-stack.adoc +:_content-type: CONCEPT [id="unmanaged-monitoring-operators_{context}"] = Support policy for monitoring Operators diff --git a/monitoring/configuring-the-monitoring-stack.adoc b/monitoring/configuring-the-monitoring-stack.adoc index 4973945672..88331769e3 100644 --- a/monitoring/configuring-the-monitoring-stack.adoc +++ b/monitoring/configuring-the-monitoring-stack.adoc @@ -69,42 +69,20 @@ include::modules/monitoring-assigning-tolerations-to-monitoring-components.adoc[ * See the link:https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[Kubernetes documentation] on taints and tolerations // Configuring persistent storage -[id="configuring-persistent-storage"] -== Configuring persistent storage - -Running cluster monitoring with persistent storage means that your metrics are stored to a persistent volume (PV) and can survive a pod being restarted or recreated. This is ideal if you require your metrics or alerting data to be guarded from data loss. For production environments, it is highly recommended to configure persistent storage. Because of the high IO demands, it is advantageous to use local storage. - -[NOTE] -==== -See xref:../scalability_and_performance/optimizing-storage.adoc#recommended-configurable-storage-technology_persistent-storage[Recommended configurable storage technology]. -==== - -[id="persistent-storage-prerequisites"] -=== Persistent storage prerequisites - -* Dedicate sufficient local persistent storage to ensure that the disk does not become full. How much storage you need depends on the number of pods. For information on system requirements for persistent storage, see xref:../scalability_and_performance/scaling-cluster-monitoring-operator.adoc#prometheus-database-storage-requirements_cluster-monitoring-operator[Prometheus database storage requirements]. - -* Make sure you have a persistent volume (PV) ready to be claimed by the persistent volume claim (PVC), one PV for each replica. Because Prometheus has two replicas and Alertmanager has three replicas, you need five PVs to support the entire monitoring stack. The PVs should be available from the Local Storage Operator. This does not apply if you enable dynamically provisioned storage. - -* Use the block type of storage. - -* xref:../storage/persistent_storage/persistent-storage-local.adoc#persistent-storage-using-local-volume[Configure local persistent storage.] -+ -[NOTE] -==== -If you use a local volume for persistent storage, do not use a raw block volume, which is described with `volumeMode: block` in the `LocalVolume` object. Prometheus cannot use raw block volumes. -==== - +include::modules/monitoring-configuring-persistent-storage.adoc[leveloffset=+1] include::modules/monitoring-configuring-a-local-persistent-volume-claim.adoc[leveloffset=+2] include::modules/monitoring-modifying-retention-time-and-size-for-prometheus-metrics-data.adoc[leveloffset=+2] [role="_additional-resources"] .Additional resources -* See xref:../monitoring/configuring-the-monitoring-stack.adoc#preparing-to-configure-the-monitoring-stack[Preparing to configure the monitoring stack] for steps to create monitoring config maps. -* xref:../monitoring/enabling-monitoring-for-user-defined-projects.adoc#enabling-monitoring-for-user-defined-projects[Enabling monitoring for user-defined projects] +* xref:../monitoring/configuring-the-monitoring-stack.adoc#creating-cluster-monitoring-configmap_configuring-the-monitoring-stack[Creating a cluster monitoring config map] +* xref:../scalability_and_performance/scaling-cluster-monitoring-operator.adoc#prometheus-database-storage-requirements_cluster-monitoring-operator[Prometheus database storage requirements] +* xref:../scalability_and_performance/optimizing-storage.adoc#recommended-configurable-storage-technology_persistent-storage[Recommended configurable storage technology] * xref:../storage/understanding-persistent-storage.adoc#understanding-persistent-storage[Understanding persistent storage] * xref:../scalability_and_performance/optimizing-storage.adoc#optimizing-storage[Optimizing storage] +* xref:../storage/persistent_storage/persistent-storage-local.adoc#persistent-storage-using-local-volume[Configure local persistent storage] +* xref:../monitoring/enabling-monitoring-for-user-defined-projects.adoc#enabling-monitoring-for-user-defined-projects[Enabling monitoring for user-defined projects] // Configuring remote write storage for Prometheus include::modules/monitoring-configuring-remote-write-storage.adoc[leveloffset=+1] diff --git a/monitoring/managing-alerts.adoc b/monitoring/managing-alerts.adoc index 57d85c4083..1e0ccc0136 100644 --- a/monitoring/managing-alerts.adoc +++ b/monitoring/managing-alerts.adoc @@ -65,6 +65,13 @@ include::modules/monitoring-creating-alert-routing-for-user-defined-projects.ado // Applying a custom Alertmanager configuration include::modules/monitoring-applying-custom-alertmanager-configuration.adoc[leveloffset=+1] +[role="_additional-resources"] +.Additional resources + +* See link:https://www.pagerduty.com/[the PagerDuty official site] for more information on PagerDuty. +* See link:https://www.pagerduty.com/docs/guides/prometheus-integration-guide/[the PagerDuty Prometheus Integration Guide] to learn how to retrieve the `service_key`. +* See link:https://prometheus.io/docs/alerting/configuration/[Alertmanager configuration] for configuring alerting through different alert receivers. + == Next steps * xref:../monitoring/reviewing-monitoring-dashboards.adoc#reviewing-monitoring-dashboards[Reviewing monitoring dashboards] diff --git a/monitoring/managing-metrics.adoc b/monitoring/managing-metrics.adoc index 02f101f692..ac4c7fb4ff 100644 --- a/monitoring/managing-metrics.adoc +++ b/monitoring/managing-metrics.adoc @@ -12,6 +12,11 @@ You can collect metrics to monitor how cluster components and your own workloads // Understanding metrics include::modules/monitoring-understanding-metrics.adoc[leveloffset=+1] +[role="_additional-resources"] +.Additional resources + +* link:https://prometheus.io/docs/instrumenting/clientlibs/[Prometheus client library documentation] + // Setting up metrics collection for user-defined projects include::modules/monitoring-setting-up-metrics-collection-for-user-defined-projects.adoc[leveloffset=+1] include::modules/monitoring-deploying-a-sample-service.adoc[leveloffset=+2]