From 023e8fa19303e60feb3f784919bd03e77b57db4e Mon Sep 17 00:00:00 2001 From: Pan Ousley Date: Fri, 18 Nov 2022 15:24:02 -0500 Subject: [PATCH] CNV-15822: adding live migration metrics --- _topic_maps/_topic_map.yml | 4 +-- modules/virt-live-migration-metrics.adoc | 27 +++++++++++++++++++ modules/virt-monitoring-vm-migration-cli.adoc | 2 +- modules/virt-querying-metrics.adoc | 22 +++++++-------- virt/live_migration/virt-live-migration.adoc | 3 ++- virt/live_migration/virt-migrate-vmi.adoc | 6 +++-- .../virt-monitor-vmi-migration.adoc | 7 +++++ .../virt-prometheus-queries.adoc | 14 +++++----- 8 files changed, 62 insertions(+), 23 deletions(-) create mode 100644 modules/virt-live-migration-metrics.adoc rename virt/{live_migration => logging_events_monitoring}/virt-monitor-vmi-migration.adoc (76%) diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index 6a33ab41f6..18426963f4 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -3573,8 +3573,6 @@ Topics: File: virt-migrate-vmi - Name: Migrating a virtual machine over a dedicated additional network File: virt-migrating-vm-on-secondary-network - - Name: Monitoring live migration - File: virt-monitor-vmi-migration - Name: Cancelling the live migration of a virtual machine instance File: virt-cancel-vmi-migration - Name: Configuring virtual machine eviction strategy @@ -3604,6 +3602,8 @@ Topics: File: virt-logs - Name: Viewing events File: virt-events + - Name: Monitoring live migration + File: virt-monitor-vmi-migration - Name: Diagnosing data volumes using events and conditions File: virt-diagnosing-datavolumes-using-events-and-conditions - Name: Viewing information about virtual machine workloads diff --git a/modules/virt-live-migration-metrics.adoc b/modules/virt-live-migration-metrics.adoc new file mode 100644 index 0000000000..638eba1937 --- /dev/null +++ b/modules/virt-live-migration-metrics.adoc @@ -0,0 +1,27 @@ +// Module included in the following assemblies: +// +// * virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc +// * virt/logging_events_monitoring/virt-prometheus-queries.adoc + +:_content-type: REFERENCE +[id="virt-live-migration-metrics_{context}"] += Live migration metrics + +The following metrics can be queried to show live migration status: + +`kubevirt_migrate_vmi_data_processed_bytes`:: The amount of guest operating system (OS) data that has migrated to the new virtual machine (VM). Type: Gauge. + +`kubevirt_migrate_vmi_data_remaining_bytes`:: The amount of guest OS data that remains to be migrated. Type: Gauge. + +`kubevirt_migrate_vmi_dirty_memory_rate_bytes`:: The rate at which memory is becoming dirty in the guest OS. Dirty memory is data that has been changed but not yet written to disk. Type: Gauge. + +`kubevirt_migrate_vmi_pending_count`:: The number of pending migrations. Type: Gauge. + +`kubevirt_migrate_vmi_scheduling_count`:: The number of scheduling migrations. Type: Gauge. + +`kubevirt_migrate_vmi_running_count`:: The number of running migrations. Type: Gauge. + +`kubevirt_migrate_vmi_succeeded_total`:: The number of successfully completed migrations. Type: Gauge. + +`kubevirt_migrate_vmi_failed_total`:: The number of failed migrations. Type: Gauge. + diff --git a/modules/virt-monitoring-vm-migration-cli.adoc b/modules/virt-monitoring-vm-migration-cli.adoc index 2b3ac83015..c810c2b1a2 100644 --- a/modules/virt-monitoring-vm-migration-cli.adoc +++ b/modules/virt-monitoring-vm-migration-cli.adoc @@ -1,6 +1,6 @@ // Module included in the following assemblies: // -// * virt/live_migration/virt-monitor-vmi-migration.adoc +// * virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc :_content-type: PROCEDURE [id="virt-monitoring-vm-migration-cli_{context}"] diff --git a/modules/virt-querying-metrics.adoc b/modules/virt-querying-metrics.adoc index 1d40b6d9ad..0090aad50f 100644 --- a/modules/virt-querying-metrics.adoc +++ b/modules/virt-querying-metrics.adoc @@ -18,7 +18,7 @@ The following examples use `topk` queries that specify a time period. If virtual The following query can identify virtual machines that are waiting for Input/Output (I/O): `kubevirt_vmi_vcpu_wait_seconds`:: -Returns the wait time (in seconds) for a virtual machine's vCPU. +Returns the wait time (in seconds) for a virtual machine's vCPU. Type: Counter. A value above '0' means that the vCPU wants to run, but the host scheduler cannot run it yet. This inability to run indicates that there is an issue with I/O. @@ -40,10 +40,10 @@ topk(3, sum by (name, namespace) (rate(kubevirt_vmi_vcpu_wait_seconds[6m]))) > 0 The following queries can identify virtual machines that are saturating the network: `kubevirt_vmi_network_receive_bytes_total`:: -Returns the total amount of traffic received (in bytes) on the virtual machine's network. +Returns the total amount of traffic received (in bytes) on the virtual machine's network. Type: Counter. `kubevirt_vmi_network_transmit_bytes_total`:: -Returns the total amount of traffic transmitted (in bytes) on the virtual machine's network. +Returns the total amount of traffic transmitted (in bytes) on the virtual machine's network. Type: Counter. .Example network traffic query [source,promql] @@ -61,10 +61,10 @@ topk(3, sum by (name, namespace) (rate(kubevirt_vmi_network_receive_bytes_total[ The following queries can identify VMs that are writing large amounts of data: `kubevirt_vmi_storage_read_traffic_bytes_total`:: -Returns the total amount (in bytes) of the virtual machine's storage-related traffic. +Returns the total amount (in bytes) of the virtual machine's storage-related traffic. Type: Counter. `kubevirt_vmi_storage_write_traffic_bytes_total`:: -Returns the total amount of storage writes (in bytes) of the virtual machine's storage-related traffic. +Returns the total amount of storage writes (in bytes) of the virtual machine's storage-related traffic. Type: Counter. .Example storage-related traffic query [source,promql] @@ -77,10 +77,10 @@ topk(3, sum by (name, namespace) (rate(kubevirt_vmi_storage_read_traffic_bytes_t === Storage snapshot data `kubevirt_vmsnapshot_disks_restored_from_source_total`:: -Returns the total number of virtual machine disks restored from the source virtual machine. +Returns the total number of virtual machine disks restored from the source virtual machine. Type: Gauge. `kubevirt_vmsnapshot_disks_restored_from_source_bytes`:: -Returns the amount of space in bytes restored from the source virtual machine. +Returns the amount of space in bytes restored from the source virtual machine. Type: Gauge. .Examples of storage snapshot data queries [source,promql] @@ -101,10 +101,10 @@ kubevirt_vmsnapshot_disks_restored_from_source_bytes{vm_name="simple-vm", vm_nam The following queries can determine the I/O performance of storage devices: `kubevirt_vmi_storage_iops_read_total`:: -Returns the amount of write I/O operations the virtual machine is performing per second. +Returns the amount of write I/O operations the virtual machine is performing per second. Type: Counter. `kubevirt_vmi_storage_iops_write_total`:: -Returns the amount of read I/O operations the virtual machine is performing per second. +Returns the amount of read I/O operations the virtual machine is performing per second. Type: Counter. .Example I/O performance query [source,promql] @@ -119,10 +119,10 @@ topk(3, sum by (name, namespace) (rate(kubevirt_vmi_storage_iops_read_total[6m]) The following queries can identify which swap-enabled guests are performing the most memory swapping: `kubevirt_vmi_memory_swap_in_traffic_bytes_total`:: -Returns the total amount (in bytes) of memory the virtual guest is swapping in. +Returns the total amount (in bytes) of memory the virtual guest is swapping in. Type: Gauge. `kubevirt_vmi_memory_swap_out_traffic_bytes_total`:: -Returns the total amount (in bytes) of memory the virtual guest is swapping out. +Returns the total amount (in bytes) of memory the virtual guest is swapping out. Type: Gauge. .Example memory swapping query [source,promql] diff --git a/virt/live_migration/virt-live-migration.adoc b/virt/live_migration/virt-live-migration.adoc index d66be63ad2..aeedf1adef 100644 --- a/virt/live_migration/virt-live-migration.adoc +++ b/virt/live_migration/virt-live-migration.adoc @@ -13,5 +13,6 @@ include::modules/virt-about-live-migration.adoc[leveloffset=+1] == Additional resources * xref:../../virt/live_migration/virt-migrate-vmi.adoc#virt-migrate-vmi[Migrating a virtual machine instance to another node] +* xref:../../virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc#virt-monitor-vmi-migration[Monitoring live migration] * xref:../../virt/live_migration/virt-live-migration-limits.adoc#virt-live-migration-limits[Live migration limiting] -* xref:../../virt/virtual_machines/virtual_disks/virt-creating-data-volumes.adoc#virt-customizing-storage-profile_virt-creating-data-volumes[Customizing the storage profile] +* xref:../../virt/virtual_machines/virtual_disks/virt-creating-data-volumes.adoc#virt-customizing-storage-profile_virt-creating-data-volumes[Customizing the storage profile] \ No newline at end of file diff --git a/virt/live_migration/virt-migrate-vmi.adoc b/virt/live_migration/virt-migrate-vmi.adoc index 2cd81bff46..ccdd45e1c2 100644 --- a/virt/live_migration/virt-migrate-vmi.adoc +++ b/virt/live_migration/virt-migrate-vmi.adoc @@ -14,10 +14,12 @@ If a virtual machine uses a host model CPU, you can perform live migration of th ==== include::modules/virt-initiating-vm-migration-web.adoc[leveloffset=+1] + include::modules/virt-initiating-vm-migration-cli.adoc[leveloffset=+1] [role="_additional-resources"] -.Additional resources: +[id="virt-migrate-vmi_additional-resources"] +== Additional resources -* xref:../../virt/live_migration/virt-monitor-vmi-migration.adoc#virt-monitor-vmi-migration[Monitoring live migration of a virtual machine instance] +* xref:../../virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc#virt-monitor-vmi-migration[Monitoring live migration] * xref:../../virt/live_migration/virt-cancel-vmi-migration.adoc#virt-cancel-vmi-migration[Cancelling the live migration of a virtual machine instance] diff --git a/virt/live_migration/virt-monitor-vmi-migration.adoc b/virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc similarity index 76% rename from virt/live_migration/virt-monitor-vmi-migration.adoc rename to virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc index 22c8d99b2a..9218a2a830 100644 --- a/virt/live_migration/virt-monitor-vmi-migration.adoc +++ b/virt/logging_events_monitoring/virt-monitor-vmi-migration.adoc @@ -16,3 +16,10 @@ You can monitor the progress of all live migrations on the xref:../../virt/virt- You can view the migration metrics of a virtual machine on the xref:../../virt/virt-web-console-overview.adoc#ui-virtualmachine-details-metrics_virt-web-console-overview[*VirtualMachine details -> Metrics* tab] in the web console. include::modules/virt-monitoring-vm-migration-cli.adoc[leveloffset=+1] + +[id="metrics_virt-monitor-vmi-migration"] +== Metrics + +You can use xref:../../virt/logging_events_monitoring/virt-prometheus-queries.adoc#virt-prometheus-queries[Prometheus queries] to monitor live migration. + +include::modules/virt-live-migration-metrics.adoc[leveloffset=+2] \ No newline at end of file diff --git a/virt/logging_events_monitoring/virt-prometheus-queries.adoc b/virt/logging_events_monitoring/virt-prometheus-queries.adoc index fdcc3f22dc..f6bc384937 100644 --- a/virt/logging_events_monitoring/virt-prometheus-queries.adoc +++ b/virt/logging_events_monitoring/virt-prometheus-queries.adoc @@ -3,15 +3,11 @@ = Prometheus queries for virtual resources include::_attributes/common-attributes.adoc[] :context: virt-prometheus-queries +:toclevels: 4 toc::[] -{VirtProductName} provides metrics for monitoring how infrastructure resources are consumed in the cluster. The metrics cover the following resources: - -* vCPU -* Network -* Storage -* Guest memory swapping +{VirtProductName} provides metrics that you can use to monitor the consumption of cluster infrastructure resources, including vCPU, network, storage, and guest memory swapping. You can also use metrics to query live migration status. Use the {product-title} monitoring dashboard to query virtualization metrics. @@ -30,8 +26,14 @@ include::modules/monitoring-querying-metrics-for-user-defined-projects-as-a-deve include::modules/virt-querying-metrics.adoc[leveloffset=+1] +include::modules/virt-live-migration-metrics.adoc[leveloffset=+1] + [id="additional-resources_virt-prometheus-queries"] [role="_additional-resources"] == Additional resources * xref:../../monitoring/monitoring-overview.adoc#monitoring-overview[Monitoring overview] + +* link:https://prometheus.io/docs/prometheus/latest/querying/basics/[Querying Prometheus] + +* link:https://prometheus.io/docs/prometheus/latest/querying/examples/[Prometheus query examples]