From 9bd4d6aed51abdfb8af9dc3278b0d7e09f02e7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CShauna=20Diaz=E2=80=9D?= Date: Thu, 8 Aug 2024 13:01:13 -0400 Subject: [PATCH] OSDOCS-11444: adds low latency MicroShift 4.17 --- _attributes/attributes-microshift.adoc | 2 + _topic_maps/_topic_map_ms.yml | 9 +- .../microshift_low_latency/_attributes | 1 + .../microshift_low_latency/images | 1 + .../microshift-low-latency.adoc | 98 +++++++++++++++++ .../microshift-workload-partitioning.adoc | 0 .../microshift_low_latency/modules | 1 + .../microshift_low_latency/snippets | 1 + modules/microshift-config-yaml.adoc | 8 +- .../microshift-install-rpms-low-latency.adoc | 35 ++++++ modules/microshift-low-latency-concept.adoc | 28 +++++ .../microshift-low-latency-config-yaml.adoc | 71 ++++++++++++ ...ow-latency-install-kernelrt-rhel-edge.adoc | 91 +++++++++++++++ ...hift-low-latency-install-kernelrt-rpm.adoc | 50 +++++++++ .../microshift-low-latency-kernelrt-conc.adoc | 16 +++ ...croshift-low-latency-prepare-workload.adoc | 94 ++++++++++++++++ ...t-low-latency-rhel-edge-blueprint-rtk.adoc | 104 ++++++++++++++++++ .../microshift-low-latency-tuned-auto.adoc | 63 +++++++++++ .../microshift-low-latency-tuned-conc.adoc | 14 +++ .../microshift-low-latency-tuned-profile.adoc | 93 ++++++++++++++++ 20 files changed, 776 insertions(+), 4 deletions(-) create mode 120000 microshift_configuring/microshift_low_latency/_attributes create mode 120000 microshift_configuring/microshift_low_latency/images create mode 100644 microshift_configuring/microshift_low_latency/microshift-low-latency.adoc rename microshift_configuring/{ => microshift_low_latency}/microshift-workload-partitioning.adoc (100%) create mode 120000 microshift_configuring/microshift_low_latency/modules create mode 120000 microshift_configuring/microshift_low_latency/snippets create mode 100644 modules/microshift-install-rpms-low-latency.adoc create mode 100644 modules/microshift-low-latency-concept.adoc create mode 100644 modules/microshift-low-latency-config-yaml.adoc create mode 100644 modules/microshift-low-latency-install-kernelrt-rhel-edge.adoc create mode 100644 modules/microshift-low-latency-install-kernelrt-rpm.adoc create mode 100644 modules/microshift-low-latency-kernelrt-conc.adoc create mode 100644 modules/microshift-low-latency-prepare-workload.adoc create mode 100644 modules/microshift-low-latency-rhel-edge-blueprint-rtk.adoc create mode 100644 modules/microshift-low-latency-tuned-auto.adoc create mode 100644 modules/microshift-low-latency-tuned-conc.adoc create mode 100644 modules/microshift-low-latency-tuned-profile.adoc diff --git a/_attributes/attributes-microshift.adoc b/_attributes/attributes-microshift.adoc index 7e1bced496..725f1a7b36 100644 --- a/_attributes/attributes-microshift.adoc +++ b/_attributes/attributes-microshift.adoc @@ -18,6 +18,8 @@ :op-system-base: RHEL :op-system-ostree-first: Red Hat Enterprise Linux for Edge (RHEL for Edge) :op-system-ostree: RHEL for Edge +:op-system-rt-kernel: Red Hat Enterprise Linux for Real Time (real-time kernel) +:op-system-rtk: real-time kernel :op-system-version: 9.4 :op-system-version-major: 9 :op-system-bundle: Red Hat Device Edge diff --git a/_topic_maps/_topic_map_ms.yml b/_topic_maps/_topic_map_ms.yml index 44a4e61f1d..f91d79cbea 100644 --- a/_topic_maps/_topic_map_ms.yml +++ b/_topic_maps/_topic_map_ms.yml @@ -397,8 +397,13 @@ Topics: File: microshift-greenboot-checking-status - Name: Configuring audit logging policies File: microshift-audit-logs-config -- Name: Workload partitioning - File: microshift-workload-partitioning +- Name: Configuring low latency + Dir: microshift_low_latency + Topics: + - Name: Configuring low latency + File: microshift-low-latency + - Name: Workload partitioning + File: microshift-workload-partitioning --- Name: Networking Dir: microshift_networking diff --git a/microshift_configuring/microshift_low_latency/_attributes b/microshift_configuring/microshift_low_latency/_attributes new file mode 120000 index 0000000000..20cc1dcb77 --- /dev/null +++ b/microshift_configuring/microshift_low_latency/_attributes @@ -0,0 +1 @@ +../../_attributes/ \ No newline at end of file diff --git a/microshift_configuring/microshift_low_latency/images b/microshift_configuring/microshift_low_latency/images new file mode 120000 index 0000000000..847b03ed05 --- /dev/null +++ b/microshift_configuring/microshift_low_latency/images @@ -0,0 +1 @@ +../../images/ \ No newline at end of file diff --git a/microshift_configuring/microshift_low_latency/microshift-low-latency.adoc b/microshift_configuring/microshift_low_latency/microshift-low-latency.adoc new file mode 100644 index 0000000000..1d05e821ff --- /dev/null +++ b/microshift_configuring/microshift_low_latency/microshift-low-latency.adoc @@ -0,0 +1,98 @@ +:_mod-docs-content-type: ASSEMBLY +[id="microshift-low-latency"] += Configuring low latency +include::_attributes/attributes-microshift.adoc[] +:context: microshift-low-latency + +toc::[] + +You can configure and tune low latency capabilities to improve application performance on edge devices. + +include::modules/microshift-low-latency-concept.adoc[leveloffset=+1] + +//additional resources for the low latency concept +[role="_additional-resources"] +.Additional resources +* link:https://docs.redhat.com/en/documentation/openshift_container_platform/4.16/html/scalability_and_performance/low-latency-tuning#cnf-understanding-low-latency_cnf-understanding-low-latency[About low latency] (OpenShift Container Platform documentation) + +include::modules/microshift-install-rpms-low-latency.adoc[leveloffset=+1] + +include::modules/microshift-low-latency-config-yaml.adoc[leveloffset=+1] + +//additional resources for the config.yaml +[role="_additional-resources"] +.Additional resources +//TODO * workload partitioning crossref here +* xref:../../microshift_configuring/microshift-using-config-tools.adoc#microshift-config-yaml_microshift-configuring[Using a YAML configuration file] +* link:https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration[KubeletConfiguration reference] (Kubernetes upstream documentation) + +//RHEL TuneD +include::modules/microshift-low-latency-tuned-conc.adoc[leveloffset=+1] + +//microshift-baseline is the name of the profile and used for automatic activation settings +//microshift-baseline-variables.conf is the file for user tweaks +include::modules/microshift-low-latency-tuned-profile.adoc[leveloffset=+2] + +//additional resources for tuned profiles +[role="_additional-resources"] +[id="additional-resources-tuned-profile_{context}"] +.Additional resources +* link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/monitoring_and_managing_system_status_and_performance/getting-started-with-tuned_monitoring-and-managing-system-status-and-performance[Getting started with TuneD] (RHEL documentation) +* link:https://www.redhat.com/sysadmin/linux-tuned-tuning-profiles[How to manage tuning profiles in Linux] (Red Hat blog) + +//microshift-baseline is the name of the profile and used for automatic activation settings +include::modules/microshift-low-latency-tuned-auto.adoc[leveloffset=+2] + +//additional resources for tuned profiles automatic activation +[role="_additional-resources"] +[id="additional-resources-tuned-auto_{context}"] +.Additional resources +* xref:../../microshift_install/microshift-greenboot.adoc#greenboot-directories-details_microshift-greenboot[Greenboot directories details] + +//RHEL real-time kernel +include::modules/microshift-low-latency-kernelrt-conc.adoc[leveloffset=+1] + +include::modules/microshift-low-latency-install-kernelrt-rpm.adoc[leveloffset=+2] + +include::modules/microshift-low-latency-install-kernelrt-rhel-edge.adoc[leveloffset=+2] + +[id="microshift-low-latency-install-kernelrt-rhel-edge-buildimage_{context}"] +== Building the {op-system-ostree-first} image with the real-time kernel + +Complete the build process by starting with the following procedure to embed {microshift-short}in a {op-system-ostree} image. Then complete the remaining steps in the installation documentation for installing {microshift-short} in a {op-system-ostree} image: + +* xref:../../microshift_install/microshift-embed-in-rpm-ostree.adoc#microshift-creating-ostree-iso_microshift-embed-in-rpm-ostree[Embedding in a {op-system-ostree} image] + +//additional resources for real-time kernel +[role="_additional-resources"] +[id="additional-resources-rtk_{context}"] +.Additional resources +* link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux_for_real_time/9[Red Hat Enterprise Linux for Real Time 9] (RHEL documentation) +* link:https://osbuild.org/docs/on-premises/installation/managing-repositories/#using-repositories-that-require-subscription[Using repositories that require subscription] (osbuild documentation) +* link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/composing_a_customized_rhel_system_image/creating-system-images-with-composer-command-line-interface_composing-a-customized-rhel-system-image#building-rhel-images-by-using-the-real-time-kernel_creating-system-images-with-composer-command-line-interface[Building RHEL images by using the real-time kernel] for more information. +* link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux_for_real_time/9/html/installing_rhel_9_for_real_time/assembly_installing-rhel-for-real-time_installing-rhel-9-for-real-time#post-installation-instructions_assembly_installing-rhel-for-real-time[Post installation instructions] (RHEL for Real Time documentation) +* xref:../../microshift_install/microshift-embed-in-rpm-ostree.adoc#microshift-embed-in-rpm-ostree[Embedding in a {op-system-ostree} image] + +* link:https://access.redhat.com/solutions/4096521[FAQ about RHEL for Real Time (kernel-rt)] +//Can likely remove KCS after 2024 as it may go out of date + +include::modules/microshift-low-latency-prepare-workload.adoc[leveloffset=+1] + +//additional resources for preparing the workload +[role="_additional-resources"] +[id="additional-resources-prep-workload_{context}"] +.Additional resources + +* link:https://docs.openshift.com/container-platform/4.16/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.html#cnf-configuring-high-priority-workload-pods_cnf-provisioning-low-latency[Disabling power saving mode for high priority pods] (Red Hat OpenShift Container Platform documentation) + +* link:https://docs.openshift.com/container-platform/4.16/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.html#cnf-disabling-cpu-cfs-quota_cnf-provisioning-low-latency[Disabling CPU CFS quota] (Red Hat OpenShift Container Platform documentation) + +* link:https://docs.openshift.com/container-platform/4.16/scalability_and_performance/low_latency_tuning/cnf-provisioning-low-latency-workloads.html#cnf-disabling-interrupt-processing-for-individual-pods_cnf-provisioning-low-latency[Disabling interrupt processing for CPUs where pinned containers are running] (Red Hat OpenShift Container Platform documentation) + +include::modules/microshift-low-latency-rhel-edge-blueprint-rtk.adoc[leveloffset=+1] + +//additional resources for workload partitioning +[role="_additional-resources"] +[id="additional-resources-wp_{context}"] +.Additional resources +* xref:../microshift_low_latency/microshift-workload-partitioning.adoc#microshift-workload-partitioning[Workload partitioning] \ No newline at end of file diff --git a/microshift_configuring/microshift-workload-partitioning.adoc b/microshift_configuring/microshift_low_latency/microshift-workload-partitioning.adoc similarity index 100% rename from microshift_configuring/microshift-workload-partitioning.adoc rename to microshift_configuring/microshift_low_latency/microshift-workload-partitioning.adoc diff --git a/microshift_configuring/microshift_low_latency/modules b/microshift_configuring/microshift_low_latency/modules new file mode 120000 index 0000000000..36719b9de7 --- /dev/null +++ b/microshift_configuring/microshift_low_latency/modules @@ -0,0 +1 @@ +../../modules/ \ No newline at end of file diff --git a/microshift_configuring/microshift_low_latency/snippets b/microshift_configuring/microshift_low_latency/snippets new file mode 120000 index 0000000000..5a3f5add14 --- /dev/null +++ b/microshift_configuring/microshift_low_latency/snippets @@ -0,0 +1 @@ +../../snippets/ \ No newline at end of file diff --git a/modules/microshift-config-yaml.adoc b/modules/microshift-config-yaml.adoc index 0d23d805c8..7e8ee3b1d2 100644 --- a/modules/microshift-config-yaml.adoc +++ b/modules/microshift-config-yaml.adoc @@ -6,17 +6,21 @@ [id="microshift-config-yaml_{context}"] = Using a YAML configuration file -At start up, {microshift-short} checks the system-wide `/etc/microshift/` directory for a configuration file named `config.yaml`. If the configuration file does not exist in the directory, the built-in default values are used to start the service. +At start up, {microshift-short} checks the system-wide `/etc/microshift/` directory for a configuration file named `config.yaml`. If the configuration file does not exist in the directory, built-in default values are used to start the service. [id="microshift-yaml-custom_{context}"] == Custom settings -To create custom configurations, you must create a `config.yaml` file in the `/etc/microshift/` directory, and then change any settings that are expected to override the defaults before starting or restarting {microshift-short}. +To create custom configurations, make a copy of the provided `config.yaml.default` file that is provided in the `/etc/microshift/` directory, renaming it `config.yaml`. Keep this file in the `/etc/microshift/` directory, and then you can change supported settings that are expected to override the defaults before starting or restarting {microshift-short}. [IMPORTANT] ==== Restart {microshift-short} after changing any configuration settings to have them take effect. The `config.yaml` file is read only when {microshift-short} starts. ==== +[id="microshift-yaml-custom-settings_{context}"] +== Separate restarts +Applications and other optional services used with your {microshift-short} cluster might also need to be restarted separately to apply configuration changes throughout the cluster. For example, when making changes to certain networking settings, you must stop and restart service and application pods to apply those changes. See each procedure for the task you are completing for more information. + [TIP] ==== If you add all of the configurations you need at the same time, you can minimize system restarts. diff --git a/modules/microshift-install-rpms-low-latency.adoc b/modules/microshift-install-rpms-low-latency.adoc new file mode 100644 index 0000000000..d3f4494723 --- /dev/null +++ b/modules/microshift-install-rpms-low-latency.adoc @@ -0,0 +1,35 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-installing-low-latency-rpm-package_{context}"] += Installing the {microshift-short} low latency RPM package + +When you install {microshift-short}, the low latency RPM package is not installed by default. You can install the low latency RPM as an optional package. + +.Prerequisites + +. You installed the {microshift-short} RPM. +. You configured workload partitioning for {microshift-short}. + +.Procedure + +* Install the low latency RPM package by running the following command: ++ +[source,terminal] +---- +$ sudo dnf install -y microshift-low-latency +---- ++ +[TIP] +==== +Wait to restart the host until after activating your TuneD profile. Restarting the host restarts {microshift-short} and CRI-O, which applies the low latency manifests and activates the TuneD profile. +==== + +.Next steps +. Configure the kubelet parameter for low latency in the {microshift-short} `config.yaml`. +. Tune your operating system, for example, configure and activate a TuneD profile. +. Optional: Configure automatic activation of your TuneD profile. +. Optional: If you are using the x86_64 architecture, install {op-system-rt-kernel}. +. Prepare your workloads for low latency. \ No newline at end of file diff --git a/modules/microshift-low-latency-concept.adoc b/modules/microshift-low-latency-concept.adoc new file mode 100644 index 0000000000..d817fdf65e --- /dev/null +++ b/modules/microshift-low-latency-concept.adoc @@ -0,0 +1,28 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: CONCEPT +[id="microshift-low-latency-concept_{context}"] += Lowering latency in {microshift-short} applications + +Latency is defined as the time from an event to the response to that event. You can use low latency configurations and tuning in a {microshift-short} cluster running in an operational or software-defined control system where an edge device has to respond quickly to an external event. You can fully optimize low latency performance by combining {microshift-short} configurations with operating system tuning and workload partitioning. + +[IMPORTANT] +==== +The CPU set for management applications, such as the {microshift-short} service, OVS, CRI-O, {microshift-short} pods, and isolated cores, must contain all-online CPUs. +==== + +[id="microshift-low-latency-workflow_{context}"] +== Workflow for configuring low latency for {microshift-short} applications +To configure low latency for applications running in a {microshift-short} cluster, you must complete the following tasks: + +Required:: +* Install the `microshift-low-latency` RPM. +* Configure workload partitioning. +* Configure the `kubelet` section of the `config.yaml` file in the `/etc/microshift/` directory. +* Configure and activate a TuneD profile. TuneD is a {op-system-base-full} service that monitors the host system and optimizes performance under certain workloads. +* Restart the host. + +Optional:: +* If you are using the x86_64 architecture, you can install link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux_for_real_time/9[Red Hat Enterprise Linux for Real Time 9]. diff --git a/modules/microshift-low-latency-config-yaml.adoc b/modules/microshift-low-latency-config-yaml.adoc new file mode 100644 index 0000000000..a99d95043f --- /dev/null +++ b/modules/microshift-low-latency-config-yaml.adoc @@ -0,0 +1,71 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-config-yaml_{context}"] += Configuration kubelet parameters and values in {microshift-short} + +The first step in enabling low latency to a {microshift-short} cluster is to add configurations to the {microshift-short} `config.yaml` file. + +.Prerequisites + +* You installed the OpenShift CLI (`oc`). +* You have root access to the cluster. +* You made a copy of the provided `config.yaml.default` file in the `/etc/microshift/` directory, and renamed it `config.yaml`. + +.Procedure + +* Add the kubelet configuration to the {microshift-short} `config.yaml` file: ++ +.Example passthrough `kubelet` configuration +[source,yaml] +---- +apiServer: +# ... +kubelet: <1> + cpuManagerPolicy: static # <2> + cpuManagerPolicyOptions: + full-pcpus-only: "true" # <3> + cpuManagerReconcilePeriod: 5s + memoryManagerPolicy: Static # <4> + topologyManagerPolicy: single-numa-node + reservedSystemCPUs: 0-1 # <5> + reservedMemory: + - limits: + memory: 1100Mi # <6> + numaNode: 0 + kubeReserved: + memory: 500Mi + systemReserved: + memory: 500Mi + evictionHard: # <7> + imagefs.available: "15%" # <8> + memory.available: "100Mi" # <9> + nodefs.available: "10%" # <10> + nodefs.inodesFree: "5%" # <11> + evictionPressureTransitionPeriod: 0s +# ... +---- +<1> If you change the CPU or memory managers in the kubelet configuration, you must remove files that cache the previous configuration. Restart the host to remove them automatically, or manually remove the `/var/lib/kubelet/cpu_manager_state` and `/var/lib/kubelet/memory_manager_state` files. +<2> The name of the policy to use. Valid values are `none` and `static`. Requires the `CPUManager` feature gate to be enabled. Default value is `none`. +<3> A set of `key=value` pairs for setting extra options that fine tune the behavior of the `CPUManager` policies. The default value is `null`. Requires both the `CPUManager` and `CPUManagerPolicyOptions` feature gates to be enabled. +<4> The name of the policy used by Memory Manager. Case-sensitive. The default value is `none`. Requires the `MemoryManager` feature gate to be enabled. +<5> Required. The `reservedSystemCPUs` value must be the inverse of the offlined CPUs because both values combined must account for all of the CPUs on the system. This parameter is essential to dividing the management and application workloads. Use this parameter to define a static CPU set for the host-level system and Kubernetes daemons, plus interrupts and timers. Then the rest of the CPUs on the system can be used exclusively for workloads. +<6> The value in `reservedMemory[0].limits.memory`, `1100` Mi in this example, is equal to `kubeReserved.memory` + `systemReserved.memory` + `evictionHard.memory.available`. +<7> The `evictionHard` parameters define under which conditions the kubelet evicts pods. When you change the default value of only one parameter for the `evictionHard` stanza, the default values of other parameters are not inherited and are set to zero. Provide all the threshold values even when you want to change just one. +<8> The `imagefs` is a filesystem that container runtimes use to store container images and container writable layers. In this example, the `evictionHard.imagefs.available` parameter means that the pod is evicted when the available space of the image filesystem is less than 15%. +<9> In this example, the `evictionHard.memory.available` parameter means that the pods are evicted when the available memory of the node drops below 100MiB. +<10> In this example, the `evictionHard.nodefs.available` parameter means that the pods are evicted when the main filesystem of the node has less than 10% available space. +<11> In this example, the `evictionHard.nodefs.inodesFree` parameter means that the pods are evicted when more than 15% of the node's main filesystem's inodes are in use. + +.Verification + +* After you complete the next steps and restart the host, you can use a root-access account to check that your settings are in the `config.yaml` file in the `/var/lib/microshift/resources/kubelet/config/` directory. + +.Next steps +. Enable workload partitioning. +. Tune your operating system. For example, configure and activate a TuneD profile. +. Optional: Configure automatic enablement of your TuneD profile. +. Optional: If you are using the x86_64 architecture, you can install {op-system-rt-kernel}. +. Prepare your {microshift-short} workloads for low latency. \ No newline at end of file diff --git a/modules/microshift-low-latency-install-kernelrt-rhel-edge.adoc b/modules/microshift-low-latency-install-kernelrt-rhel-edge.adoc new file mode 100644 index 0000000000..7670ce3e11 --- /dev/null +++ b/modules/microshift-low-latency-install-kernelrt-rhel-edge.adoc @@ -0,0 +1,91 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-install-kernelrt-rhel-edge_{context}"] += Installing the {op-system-rt-kernel} in a {op-system-ostree-first} image + +You can include the real-time kernel in a {op-system-ostree} image deployment using image builder. The following example blueprint sections include references gathered from the previous steps required to configure low latency for a {microshift-short} cluster. + +.Prerequisites +* You have a Red Hat subscription enabled on the host that includes {op-system-rt-kernel}. +* You are using the x86_64 architecture. +* You configured `osbuild` to use the `kernel-rt` repository. + +[IMPORTANT] +==== +A subscription that includes the {op-system-rtk} must be enabled on the host used to build the commit. +==== + +.Procedure + +* Add the following example blueprint sections to your complete installation blueprint for installing the real-time kernel in a {op-system-ostree} image: ++ +.Example blueprint snippet for the real-time kernel +[source,text] +---- +[[packages]] +name = "microshift-low-latency" +version = "*" + +# Kernel RT is supported only on the x86_64 architecture +[customizations.kernel] +name = "kernel-rt" + +[customizations.services] +enabled = ["microshift", "microshift-tuned"] + +[[customizations.files]] +path = "/etc/microshift/config.yaml" +data = """ +kubelet: + cpuManagerPolicy: static + cpuManagerPolicyOptions: + full-pcpus-only: "true" + cpuManagerReconcilePeriod: 5s + memoryManagerPolicy: Static + topologyManagerPolicy: single-numa-node + reservedSystemCPUs: 0-1 + reservedMemory: + - limits: + memory: 1100Mi + numaNode: 0 + kubeReserved: + memory: 500Mi + systemReserved: + memory: 500Mi + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s +""" + +[[customizations.files]] +path = "/etc/tuned/microshift-baseline-variables.conf" +data = """ +# Isolated cores should be complementary to the kubelet configuration reserved CPUs. +# Isolated and reserved CPUs must contain all online CPUs. +# Core #3 is for testing offlining, therefore it is skipped. +isolated_cores=2,4-5 +hugepages_size=2M +hugepages=10 +additional_args=test1=on test2=true dummy +offline_cpu_set=3 +""" + +[[customizations.files]] +path = "/etc/microshift/tuned.yaml" +data = """ +profile: microshift-baseline +reboot_after_apply: True +""" +---- + +.Next steps +. Complete the image building process. +. If you have not completed the previous steps for enabling low latency for your {microshift-short} cluster, do so now. Update the blueprint with the information gathered in those steps. +. If you have not configured workload partitioning, do so now. +. Prepare your {microshift-short} workloads for low latency. \ No newline at end of file diff --git a/modules/microshift-low-latency-install-kernelrt-rpm.adoc b/modules/microshift-low-latency-install-kernelrt-rpm.adoc new file mode 100644 index 0000000000..7cd45253a4 --- /dev/null +++ b/modules/microshift-low-latency-install-kernelrt-rpm.adoc @@ -0,0 +1,50 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-install-kernelrt_{context}"] += Installing the {op-system-rt-kernel} + +Although the real-time kernel is not necessary for low latency workloads, using the {op-system-rtk} can optimize low latency performance. You can install it on a host using RPM packages, and include it in a {op-system-ostree-first} image deployment. + +.Prerequisites + +* You have a Red Hat subscription that includes {op-system-rt-kernel}. For example, your host machine is registered and Red Hat Enterprise Linux (RHEL) is attached to a RHEL for Real Time subscription. +* You are using x86_64 architecture. + +.Procedure + +. Enable the {op-system-rtk} repository by running the following command: ++ +[source,terminal] +---- +$ sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms +---- + +. Install the real-time kernel by running the following command: ++ +[source,terminal] +---- +$ sudo dnf install -y kernel-rt +---- + +. Query the real-time kernel version by running the following command: ++ +[source,terminal] +---- +$ RTVER=$(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel-rt | sort | tail -1) +---- ++ +. Make a persistent change in GRUB that designates the real-time kernel as the default kernel by running the following command: ++ +[source,terminal] +---- +$ sudo grubby --set-default="/boot/vmlinuz-${RTVER}+rt" +---- + +. Restart the host to activate the real-time kernel. + +.Next steps +. Prepare your {microshift-short} workloads for low latency. +. Optional: Use a blueprint to install the real-time kernel in a {op-system-ostree} image. diff --git a/modules/microshift-low-latency-kernelrt-conc.adoc b/modules/microshift-low-latency-kernelrt-conc.adoc new file mode 100644 index 0000000000..3699028d36 --- /dev/null +++ b/modules/microshift-low-latency-kernelrt-conc.adoc @@ -0,0 +1,16 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: CONCEPT +[id="microshift-low-latency-kernelrt-conc_{context}"] += Using Red Hat Enterprise Linux for Real Time + +If your workload has stringent low-latency determinism requirements for core kernel features such as interrupt handling and process scheduling in the microsecond (μs) range, you can use the {op-system-rt-kernel}. The goal of the {op-system-rtk} is consistent, low-latency determinism that offers predictable response times. + +When considering system tuning, consider the following factors: + +* System tuning is just as important when using the {op-system-rtk} as it is for the standard kernel. +* Installing the {op-system-rtk} on an untuned system running the standard kernel supplied as part of the RHEL 9 release is not likely to result in any noticeable benefit. +* Tuning the standard kernel yields 90% of possible latency gains. +* The {op-system-rtk} provides the last 10% of latency reduction required by the most demanding workloads. diff --git a/modules/microshift-low-latency-prepare-workload.adoc b/modules/microshift-low-latency-prepare-workload.adoc new file mode 100644 index 0000000000..bf7996ae82 --- /dev/null +++ b/modules/microshift-low-latency-prepare-workload.adoc @@ -0,0 +1,94 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-prepare-workload_{context}"] += Preparing a {microshift-short} workload for low latency + +To take advantage of low latency, workloads running on {microshift-short} must have the `microshift-low-latency` container runtime configuration set by using the `RuntimeClass` feature. The CRI-O `RuntimeClass` object is installed with the `microshift-low-latency` RPM, so only the pod annotations need to be configured. + +.Prerequisites +* You installed the `microshift-low-latency` RPM package. +* You configured workload partitioning. + +.Procedure + +* Use the following example to set the following annotations in the pod spec: ++ +[source,text] +---- +cpu-load-balancing.crio.io: "disable" +irq-load-balancing.crio.io: "disable" +cpu-quota.crio.io: "disable" +cpu-load-balancing.crio.io: "disable" +cpu-freq-governor.crio.io: "" +---- ++ +.Example pod that runs `oslat` test: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: oslat + annotations: + cpu-load-balancing.crio.io: "disable" # <1> + irq-load-balancing.crio.io: "disable" # <2> + cpu-quota.crio.io: "disable" # <3> + cpu-c-states.crio.io: "disable" # <4> + cpu-freq-governor.crio.io: "" # <5> +spec: + runtimeClassName: microshift-low-latency # <6> + containers: + - name: oslat + image: quay.io/container-perf-tools/oslat + imagePullPolicy: Always + resources: + requests: + memory: "400Mi" + cpu: "2" + limits: + memory: "400Mi" + cpu: "2" + env: + - name: tool + value: "oslat" + - name: manual + value: "n" + - name: PRIO + value: "1" + - name: delay + value: "0" + - name: RUNTIME_SECONDS + value: "60" + - name: TRACE_THRESHOLD + value: "" + - name: EXTRA_ARGS + value: "" + securityContext: + privileged: true + capabilities: + add: + - SYS_NICE + - IPC_LOCK + +---- +<1> Disables the CPU load balancing for the pod. +<2> Opts the pod out of interrupt handling (IRQ). +<3> Disables the CPU completely fair scheduler (CFS) quota at the pod run time. +<4> Enables or disables C-states for each CPU. Set the value to `disable` to provide the best performance for a high-priority pod. +<5> Sets the `cpufreq` governor for each CPU. The `performance` governor is recommended for high-priority workloads. +<6> The `runtimeClassName` must match the name of the performance profile configured in the cluster. For example, `microshift-low-latency`. ++ +[NOTE] +==== +Disable CPU load balancing only when the CPU manager static policy is enabled and for pods with guaranteed QoS that use whole CPUs. Otherwise, disabling CPU load balancing can affect the performance of other containers in the cluster. +==== ++ +[IMPORTANT] +==== +For the pod to have the `Guaranteed` QoS class, it must have the same values of CPU and memory in requests and limits. See link:https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#guaranteed[Guaranteed] (Kubernetes upstream documentation) +==== + diff --git a/modules/microshift-low-latency-rhel-edge-blueprint-rtk.adoc b/modules/microshift-low-latency-rhel-edge-blueprint-rtk.adoc new file mode 100644 index 0000000000..30a8555c07 --- /dev/null +++ b/modules/microshift-low-latency-rhel-edge-blueprint-rtk.adoc @@ -0,0 +1,104 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: REFERENCE +[id="microshift-low-latency-blueprint-rhel-edge-rtk_{context}"] += Reference blueprint for installing {op-system-rt-kernel} in a {op-system-ostree} image + +An image blueprint is a persistent definition of the required image customizations that enable you to create multiple builds. Instead of reconfiguring the blueprint for each image build, you can edit, rebuild, delete, and save the blueprint so that you can keep rebuilding images from it. + +.Example blueprint used to install the real-time kernel in a {op-system-ostree} image +[source,text] +---- +name = "microshift-low-latency" +description = "RHEL 9.4 and MicroShift configured for low latency" +version = "0.0.1" +modules = [] +groups = [] +distro = "rhel-94" + +[[packages]] +name = "microshift" +version = "*" + +[[packages]] +name = "microshift-greenboot" +version = "*" + +[[packages]] +name = "microshift-networking" +version = "*" + +[[packages]] +name = "microshift-selinux" +version = "*" + +[[packages]] +name = "microshift-low-latency" +version = "*" + +# Kernel RT is only available for x86_64 +[customizations.kernel] +name = "kernel-rt" + +[customizations.services] +enabled = ["microshift", "microshift-tuned"] + +[customizations.firewall] +ports = ["22:tcp", "80:tcp", "443:tcp", "5353:udp", "6443:tcp", "30000-32767:tcp", "30000-32767:udp"] + +[customizations.firewall.services] +enabled = ["mdns", "ssh", "http", "https"] + +[[customizations.firewall.zones]] +name = "trusted" +sources = ["10.42.0.0/16", "169.254.169.1"] + +[[customizations.files]] +path = "/etc/microshift/config.yaml" +data = """ +kubelet: + cpuManagerPolicy: static + cpuManagerPolicyOptions: + full-pcpus-only: "true" + cpuManagerReconcilePeriod: 5s + memoryManagerPolicy: Static + topologyManagerPolicy: single-numa-node + reservedSystemCPUs: 0-1 + reservedMemory: + - limits: + memory: 1100Mi + numaNode: 0 + kubeReserved: + memory: 500Mi + systemReserved: + memory: 500Mi + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s +""" + +[[customizations.files]] +path = "/etc/tuned/microshift-baseline-variables.conf" +data = """ +# Isolated cores should be complementary to the kubelet configuration reserved CPUs. +# Isolated and reserved CPUs must contain all online CPUs. +# Core #3 is for testing offlining, therefore it is skipped. +isolated_cores=2,4-5 +hugepages_size=2M +hugepages=10 +additional_args=test1=on test2=true dummy +offline_cpu_set=3 +""" + +[[customizations.files]] +path = "/etc/microshift/tuned.yaml" +data = """ +profile: microshift-baseline +reboot_after_apply: True +""" +---- \ No newline at end of file diff --git a/modules/microshift-low-latency-tuned-auto.adoc b/modules/microshift-low-latency-tuned-auto.adoc new file mode 100644 index 0000000000..9d81c3f68d --- /dev/null +++ b/modules/microshift-low-latency-tuned-auto.adoc @@ -0,0 +1,63 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-tuned-auto-activate_{context}"] += Automatically enable the {microshift-short} TuneD profile + +Included in the `microshift-low-latency` RPM package is a systemd service that you can configure to automatically enable a TuneD profile when the system starts. This ability is particularly useful if you are installing {microshift-short} in a large fleet of devices. + +.Prerequisites + +. You installed the microshift-low-latency RPM package on the host. +. You enabled low latency in the {microshift-short} `config.yaml`. +. You created a TuneD profile. +. You configured the `microshift-baseline-variables.conf` file. + +.Procedure + +. Configure the `tuned.yaml` in the `/etc/microshift/` directory, for example: ++ +.Example tuned.yaml +[source,yaml] +---- +profile: microshift-baseline <1> +reboot_after_apply: True <2> +---- +<1> Controls which TuneD profile is activated. In this example, the name of the profile is `microshift-baseline`. +<2> Controls whether the host must be rebooted after applying the profile. Valid values are `True` and `False`. For example, use the `True` setting to automatically restart the host after a new `ostree` commit is deployed. ++ +[IMPORTANT] +==== +The host is restarted when the `microshift-tuned.service` runs, but it does not restart the system when a new commit is deployed. You must restart the host to enable a new commit, then the system starts again when the `microshift-tuned.service` runs on that boot and detects changes to profiles and variables. + +This double-boot can effect rollbacks. Ensure that you adjust the number of reboots in greenboot that are allowed before rollback when using automatic profile activation. For example, if 3 reboots are allowed before a rollback in greenboot, increase that number to 4. See the "Additional resources" list for more information. +==== + +. Enable the `microshift-tuned.service` to run on each system start by entering the following command: ++ +[source,terminal] +---- +$ sudo systemctl enable microshift-tuned.service +---- ++ +[IMPORTANT] +==== +If you set `reboot_after_apply` to `True`, ensure that a TuneD profile is active and that no other profiles have been activated outside of the {microshift-short} service. Otherwise, starting the `microshift-tuned.service` results in a host reboot. +==== + +. Start the `microshift-tuned.service` by running the following command: ++ +[source,terminal] +---- +$ sudo systemctl start microshift-tuned.service +---- ++ +[NOTE] +==== +The `microshift-tuned.service` uses collected checksums to detect changes to selected TuneD profiles and variables. If there are no checksums on the disk, the service activates the TuneD profile and restarts the host. Expect a host restart when first starting the `microshift-tuned.service`. +==== + +.Next steps +* Optional: If you are using the x86_64 architecture, you can install {op-system-rt-kernel}. \ No newline at end of file diff --git a/modules/microshift-low-latency-tuned-conc.adoc b/modules/microshift-low-latency-tuned-conc.adoc new file mode 100644 index 0000000000..63ece3c65e --- /dev/null +++ b/modules/microshift-low-latency-tuned-conc.adoc @@ -0,0 +1,14 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: CONCEPT +[id="microshift-low-latency-tuned-conc_{context}"] += Tuning Red Hat Enterprise Linux 9 + +As a {op-system-base-full} system administrator, you can use the TuneD service to optimize the performance profile of {op-system} for a variety of use cases. TuneD monitors and optimizes system performance under certain workloads, including latency performance. + +* Use TuneD profiles to tune your system for different use cases, such as deploying a low-latency {microshift-short} cluster. +* You can modify the rules defined for each profile and customize tuning for a specific device. +* When you switch to another profile or deactivate TuneD, all changes made to the system settings by the previous profile revert back to their original state. +* You can also configure TuneD to react to changes in device usage and adjusts settings to improve performance of active devices and reduce power consumption of inactive devices. \ No newline at end of file diff --git a/modules/microshift-low-latency-tuned-profile.adoc b/modules/microshift-low-latency-tuned-profile.adoc new file mode 100644 index 0000000000..6c897cd27f --- /dev/null +++ b/modules/microshift-low-latency-tuned-profile.adoc @@ -0,0 +1,93 @@ +// Module included in the following assemblies: +// +// microshift_configuring/microshift_low_latency/microshift-low-latency.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-low-latency-tuned-profile_{context}"] += Configuring the {microshift-short} TuneD profile + +Configure a TuneD profile for your host to use low latency with {microshift-short} workloads using the `microshift-baseline-variables.conf` configuration file provided in the {op-system-base-full} `/etc/tuned/` host directory after you install the `microshift-low-latency` RPM package. + +.Prerequisites + +* You have root access to the cluster. +* You installed the `microshift-low-latency` RPM package. +* Your {op-system} host has TuneD installed. See link:https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/monitoring_and_managing_system_status_and_performance/getting-started-with-tuned_monitoring-and-managing-system-status-and-performance#the-location-of-tuned-profiles_getting-started-with-tuned[Getting started with TuneD] (RHEL documentation). + +.Procedure + +. You can use the default `microshift-baseline-variables.conf` TuneD profile in the `/etc/tuned/` directory profile, or create your own to add more tunings. ++ +.Example `microshift-baseline-variables.conf` TuneD profile +[source,text] +---- +# Isolate cores 2-7 for running application workloads +isolated_cores=2-7 # <1> + +# Size of the hugepages +hugepages_size=2M # <2> + +# Number of hugepages +hugepages=0 + +# Additional kernel arguments +additional_args= # <3> + +# CPU set to be offlined +offline_cpu_set= # <4> +---- ++ +-- +<1> Controls which cores should be isolated. By default, 1 core per socket is reserved in {microshift-short} for housekeeping . The other cores are isolated. Valid values are a core list or range. You can isolate any range, for example: `isolated_cores=2,4-7` or `isolated_cores=2-23`. ++ +[IMPORTANT] +==== +You must keep only one `isolated_cores=` variable. +==== ++ +[NOTE] +==== +The Kubernetes CPU manager can use any CPU to run the workload except the reserved CPUs defined in the kubelet configuration. For this reason it is best that: + +* The sum of the kubelet's reserved CPUs and isolated cores include all online CPUs. +* Isolated cores are complementary to the reserved CPUs defined in the kubelet configuration. +==== ++ +<2> Size of the hugepages. Valid values are 2M or 1G. +<3> Additional kernel arguments, for example, `additional_args=console=tty0 console=ttyS0,115200`. +<4> The CPU set to be offlined. ++ +[IMPORTANT] +==== +Must not overlap with `isolated_cores`. +==== +-- + +. Enable the profile or make changes active, by running the following command: ++ +[source,terminal] +---- +$ sudo tuned-adm profile microshift-baseline +---- + +. Reboot the host to make kernel arguments active. + +.Verification + +* Optional: You can read the `/proc/cmdline` file that contains the arguments given to the currently running kernel on start. ++ +[source,terminal] +---- +$ cat /proc/cmdline +---- ++ +.Example output +[source,text] +---- +BOOT_IMAGE=(hd0,msdos2)/ostree/rhel-7f82ccd9595c3c70af16525470e32c6a81c9138c4eae6c79ab86d5a2d108d7fc/vmlinuz-5.14.0-427.31.1.el9_4.x86_64+rt crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M rd.lvm.lv=rhel/root fips=0 console=ttyS0,115200n8 root=/dev/mapper/rhel-root rw ostree=/ostree/boot.1/rhel/7f82ccd9595c3c70af16525470e32c6a81c9138c4eae6c79ab86d5a2d108d7fc/0 skew_tick=1 tsc=reliable rcupdate.rcu_normal_after_boot=1 nohz=on nohz_full=2,4-5 rcu_nocbs=2,4-5 tuned.non_isolcpus=0000000b intel_pstate=disable nosoftlockup hugepagesz=2M hugepages=10 +---- + +.Next steps +. Prepare your {microshift-short} workloads for low latency. +. Optional: Configure automatic enablement of your TuneD profile. +. Optional: If you are using the x86_64 architecture, you can install {op-system-rt-kernel}.