diff --git a/_topic_maps/_topic_map.yml b/_topic_maps/_topic_map.yml index fb436b6ed0..817b66f7a1 100644 --- a/_topic_maps/_topic_map.yml +++ b/_topic_maps/_topic_map.yml @@ -508,6 +508,7 @@ Topics: File: configuring-firewall - Name: Enabling Linux control group version 2 (cgroup v2) File: enabling-cgroup-v2 + Distros: openshift-enterprise - Name: Validating an installation File: validating-an-installation Distros: openshift-origin,openshift-enterprise @@ -2273,10 +2274,10 @@ Topics: - Name: Configuring your cluster to place pods on overcommited nodes File: nodes-cluster-overcommit Distros: openshift-enterprise,openshift-origin - - Name: Enabling Linux control group version 2 (cgroup v2) + - Name: Configuring the Linux cgroup version on your nodes File: nodes-cluster-cgroups-2 Distros: openshift-enterprise - - Name: Configuring the Linux cgroup on your nodes + - Name: Configuring the Linux cgroup version on your nodes File: nodes-cluster-cgroups-okd Distros: openshift-origin - Name: Enabling features using FeatureGates diff --git a/installing/install_config/enabling-cgroup-v2.adoc b/installing/install_config/enabling-cgroup-v2.adoc index fb81194c2c..78f1e0acd1 100644 --- a/installing/install_config/enabling-cgroup-v2.adoc +++ b/installing/install_config/enabling-cgroup-v2.adoc @@ -8,12 +8,11 @@ toc::[] ifndef::openshift-origin[] -You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster by editing the `node.config` object. Enabling cgroup v2 in {product-title} disables all cgroups version 1 controllers and hierarchies in your cluster. cgroup v1 is enabled by default. +By default, {product-title} uses link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[Linux control group version 1] (cgroup v1) in your cluster. You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) upon installation. Enabling cgroup v2 in {product-title} disables all cgroup version 1 controllers and hierarchies in your cluster. cgroup v2 is the next version of the Linux cgroup API. cgroup v2 offers several improvements over cgroup v1, including a unified hierarchy, safer sub-tree delegation, new features such as link:https://www.kernel.org/doc/html/latest/accounting/psi.html[Pressure Stall Information], and enhanced resource management and isolation. -:FeatureName: {product-title} cgroups version 2 support -include::snippets/technology-preview.adoc[leveloffset=+0] +You can switch between cgroup v1 and cgroup v2, as needed, by editing the the `node.config` object. For more information, see "Configuring the Linux cgroup on your nodes" in the "Additional resources" of this section. endif::openshift-origin[] ifdef::openshift-origin[] @@ -37,5 +36,5 @@ endif::openshift-origin[] .Additional resources -* xref:../../nodes/clusters/nodes-cluster-enabling-features.adoc#nodes-cluster-enabling[Enabling OpenShift Container Platform features using FeatureGates] * xref:../../installing/index.adoc#ocp-installation-overview[OpenShift Container Platform installation overview] +* xref:../../nodes/clusters/nodes-cluster-cgroups-2.adoc#nodes-clusters-cgroups-2_nodes-cluster-cgroups-2[Configuring the Linux cgroup on your nodes] diff --git a/modules/nodes-clusters-cgroups-2-install.adoc b/modules/nodes-clusters-cgroups-2-install.adoc index ac4ce60093..f4da4d2408 100644 --- a/modules/nodes-clusters-cgroups-2-install.adoc +++ b/modules/nodes-clusters-cgroups-2-install.adoc @@ -22,17 +22,5 @@ metadata: cgroupMode: "v2" ---- -. Create or edit the `FeatureGate` object to enable the `TechPreviewNoUpgrade` feature set: -+ -[source,yaml] ----- -apiVersion: config.openshift.io/v1 -kind: FeatureGate -metadata: - name: cluster - spec: - featureSet: "TechPreviewNoUpgrade" ----- - . Proceed with the installation as usual. diff --git a/modules/nodes-clusters-cgroups-2.adoc b/modules/nodes-clusters-cgroups-2.adoc index 65bb85f549..f6fa1aa452 100644 --- a/modules/nodes-clusters-cgroups-2.adoc +++ b/modules/nodes-clusters-cgroups-2.adoc @@ -12,34 +12,23 @@ endif::[] :_content-type: PROCEDURE [id="nodes-clusters-cgroups-2_{context}"] -= Configuring Linux cgroup v2 += Configuring Linux cgroup ifdef::post[] -You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster by editing the `node.config` object. Enabling cgroup v2 in {product-title} disables all cgroups version 1 controllers and hierarchies in your cluster. cgroup v1 is enabled by default. +link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 1] (cgroup v1) is enabled by default. You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster by editing the `node.config` object. Enabling cgroup v2 in {product-title} disables all cgroup version 1 controllers and hierarchies in your cluster. cgroup v2 is the next version of the Linux cgroup API. cgroup v2 offers several improvements over cgroup v1, including a unified hierarchy, safer sub-tree delegation, new features such as link:https://www.kernel.org/doc/html/latest/accounting/psi.html[Pressure Stall Information], and enhanced resource management and isolation. -[IMPORTANT] -==== -{product-title} cgroups version 2 support is a Technology Preview feature only. Technology Preview features -are not supported with Red Hat production service level agreements (SLAs) and -might not be functionally complete. Red Hat does not recommend using them -in production. These features provide early access to upcoming product -features, enabling customers to test functionality and provide feedback during -the development process. - -For more information about the support scope of Red Hat Technology Preview features, see link:https://access.redhat.com/support/offerings/techpreview/[Technology Preview Features Support Scope]. -==== +You can change between cgroup v1 and cgroup v2, as needed. For more information, see "Configuring the Linux cgroup on your nodes" in the "Additional resources" of this section. endif::post[] ifdef::nodes[] -You enable cgroup v2 by editing the `node.config` object. +You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[Linux control group version 1] (cgroup v1) or link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) by editing the `node.config` object. The default is cgroup v1. endif::nodes[] .Prerequisites * You have a running {product-title} cluster that uses version 4.12 or later. * You are logged in to the cluster as a user with administrative privileges. -* You have enabled the `TechPreviewNoUpgrade` feature set by using the feature gates. .Procedure @@ -52,6 +41,7 @@ endif::nodes[] $ oc edit nodes.config/cluster ---- +ifdef::post[] .. Add `spec.cgroupMode: "v2"`: + .Example `node.config` object @@ -80,7 +70,38 @@ spec: ... ---- <1> Enables cgroup v2. +endif::post[] +ifdef::nodes[] +.. Edit the `spec.cgroupMode` parameter: ++ +.Example `node.config` object +[source,yaml] +---- +apiVersion: config.openshift.io/v1 +kind: Node +metadata: + annotations: + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true" + release.openshift.io/create-only: "true" + creationTimestamp: "2022-07-08T16:02:51Z" + generation: 1 + name: cluster + ownerReferences: + - apiVersion: config.openshift.io/v1 + kind: ClusterVersion + name: version + uid: 36282574-bf9f-409e-a6cd-3032939293eb + resourceVersion: "1865" + uid: 0c0f7a4c-4307-4187-b591-6155695ac85b +spec: + cgroupMode: "v2" <1> +... +---- +<1> Specify `v2` to enable cgroup v2 or `v1` for cgroup v1. +endif::nodes[] .Verification @@ -101,15 +122,16 @@ NAME GENERATEDBYCONTROLLER 01-master-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-container-runtime 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 01-worker-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m -97-master-generated-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 3m <1> -99-worker-generated-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 3m +97-master-generated-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m +99-worker-generated-kubelet 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-master-ssh 3.2.0 40m 99-worker-generated-registries 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m 99-worker-ssh 3.2.0 40m +rendered-master-23d4317815a5f854bd3553d689cfe2e9 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 10s <1> rendered-master-23e785de7587df95a4b517e0647e5ab7 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m rendered-worker-5d596d9293ca3ea80c896a1191735bb1 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 33m -worker-enable-cgroups-v2 3.2.0 10s +rendered-worker-dcc7f1b92892d34db74d6832bcc9ccd4 52dd3ba6a9a527fc3ab42afac8d12b693534c8c9 3.2.0 10s ---- <1> New machine configs are created, as expected. @@ -120,7 +142,26 @@ worker-enable-cgroups-v2 $ oc describe mc ---- + -.Example output +ifdef::nodes[] +.Example output for cgroup v1 +[source,terminal] +---- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: 05-worker-kernelarg-selinuxpermissive +spec: + kernelArguments: + systemd.unified_cgroup_hierarchy=0 <1> + systemd.legacy_systemd_cgroup_controller=1 <2> +---- +<1> Enables cgroup v1 in systemd. +<2> Disables cgroup v2. ++ +endif::nodes[] +.Example output for cgroup v2 [source,terminal] ---- apiVersion: machineconfiguration.openshift.io/v1 @@ -136,7 +177,7 @@ spec: - psi=1 <3> ---- <1> Enables cgroup v2 in systemd. -<2> Disables cgroups v1. +<2> Disables cgroup v1. <3> Enables the Linux Pressure Stall Information (PSI) feature. . Check the nodes to see that scheduling on the nodes is disabled. This indicates that the change is being applied: @@ -150,7 +191,7 @@ $ oc get nodes [source,terminal] ---- NAME STATUS ROLES AGE VERSION -ci-ln-fm1qnwt-72292-99kt6-master-0 Ready master 58m v1.25.0 +ci-ln-fm1qnwt-72292-99kt6-master-0 Ready,SchedulingDisabled master 58m v1.25.0 ci-ln-fm1qnwt-72292-99kt6-master-1 Ready master 58m v1.25.0 ci-ln-fm1qnwt-72292-99kt6-master-2 Ready master 58m v1.25.0 ci-ln-fm1qnwt-72292-99kt6-worker-a-h5gt4 Ready,SchedulingDisabled worker 48m v1.25.0 @@ -172,6 +213,7 @@ $ oc debug node/ sh-4.4# chroot /host ---- +ifdef::post[] . Check that the `sys/fs/cgroup/cgroup2fs` file is present on your nodes. This file is created by cgroup v2: + [source,terminal] @@ -184,6 +226,27 @@ $ stat -c %T -f /sys/fs/cgroup ---- cgroup2fs ---- +endif::post[] +ifdef::nodes[] +. Check that the `sys/fs/cgroup/cgroup2fs` or `sys/fs/cgroup/tmpfs` file is present on your nodes: ++ +[source,terminal] +---- +$ stat -c %T -f /sys/fs/cgroup +---- ++ +.Example output for cgroup v1 +[source,terminal] +---- +tmp2fs +---- ++ +.Example output for cgroup v2 +[source,terminal] +---- +cgroup2fs +---- +endif::nodes[] ifeval::["{context}" == "nodes-cluster-cgroups-2"] :!nodes: diff --git a/modules/nodes-clusters-cgroups-okd-configure.adoc b/modules/nodes-clusters-cgroups-okd-configure.adoc index 0779621bb4..0ef5f5a8f8 100644 --- a/modules/nodes-clusters-cgroups-okd-configure.adoc +++ b/modules/nodes-clusters-cgroups-okd-configure.adoc @@ -3,6 +3,7 @@ // * nodes/clusters/nodes-cluster-cgroups-okd.adoc // * post_installation_configuration/cluster-tasks.adoc + ifeval::["{context}" == "nodes-cluster-cgroups-2"] :node: endif::[] @@ -10,17 +11,23 @@ ifeval::["{context}" == "post-install-cluster-tasks"] :post: endif::[] +ifdef::post[] :_content-type: PROCEDURE [id="nodes-clusters-cgroups-okd-configure_{context}"] -= Configuring Linux cgroup += Configuring the Linux cgroup version on your nodes -ifdef::post[] -By default, {product-title} uses link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster. You can switch to link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[Linux control group version 1] (cgroup v1), if needed. +By default, {product-title} uses link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster. You can switch to link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[Linux control group version 1] (cgroup v1), if needed, by using a machine config. Enabling cgroup v1 in {product-title} disables the cgroup v2 controllers and hierarchies in your cluster. cgroup v2 is the next version of the kernel link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/ch01[control group] and offers multiple improvements. However, it can have some unwanted effects on your nodes. endif::post[] -You can configure whether your cluster uses cgroup v1 or cgroup v2 by editing the `node.config` object. Enabling the other version of cgroup in {product-title} disables the current cgroup controllers and hierarchies in your cluster. +ifdef::node[] +:_content-type: PROCEDURE +[id="nodes-clusters-cgroups-okd-configure_{context}"] += Configuring Linux cgroup + +You can switch to Linux control group version 1 (cgroup v1), if needed, by using a machine config. Enabling cgroup v1 in {product-title} disables the cgroup v2 controllers and hierarchies in your cluster. +endif::node[] .Prerequisites * Have administrative privilege to a working {product-title} cluster. @@ -102,9 +109,23 @@ ip-10-0-153-11.ec2.internal Ready worker 28m v1.25.0 ip-10-0-153-150.ec2.internal Ready master 34m v1.25.0 ---- + -You can see that The command disables scheduling on each worker node. +You can see that the command disables scheduling on each worker node. -. Check that the sys/fs/cgroup/cgroup2fs file has been moved to the `tmpfs` file system: +. After a node returns to the `Ready` state, start a debug session for that node: ++ +[source,terminal] +---- +$ oc debug node/ +---- + +. Set `/host` as the root directory within the debug shell: ++ +[source,terminal] +---- +sh-4.4# chroot /host +---- + +. Check that the `sys/fs/cgroup/cgroup2fs` file has been moved to the `tmpfs` file system: + [source,terminal] ---- diff --git a/modules/nodes-nodes-kernel-arguments.adoc b/modules/nodes-nodes-kernel-arguments.adoc index 18209cdda2..058152ce48 100644 --- a/modules/nodes-nodes-kernel-arguments.adoc +++ b/modules/nodes-nodes-kernel-arguments.adoc @@ -22,12 +22,8 @@ Examples of kernel arguments you could set include: ifndef::openshift-origin[] * **systemd.unified_cgroup_hierarchy**: Enables link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2). cgroup v2 is the next version of the kernel link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/ch01[control group] and offers multiple improvements. -+ --- -:FeatureName: {product-title} cgroups version 2 support -include::snippets/technology-preview.adoc[leveloffset=+1] --- endif::openshift-origin[] + ifdef::openshift-origin[] * **systemd.unified_cgroup_hierarchy**: Configures the version of Linux control group that is installed on your nodes: link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[cgroup v1] or link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[cgroup v2]. cgroup v2 is the next version of the kernel link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/ch01[control group] and offers multiple improvements. However, it can have some unwanted effects on your nodes. + diff --git a/nodes/clusters/nodes-cluster-cgroups-2.adoc b/nodes/clusters/nodes-cluster-cgroups-2.adoc index 48d0bdd719..6c418ea58c 100644 --- a/nodes/clusters/nodes-cluster-cgroups-2.adoc +++ b/nodes/clusters/nodes-cluster-cgroups-2.adoc @@ -1,18 +1,15 @@ :_content-type: ASSEMBLY :context: nodes-cluster-cgroups-2 [id="nodes-cluster-cgroups-2"] -= Enabling Linux control group version 2 (cgroup v2) += Configuring the Linux cgroup version on your nodes include::_attributes/common-attributes.adoc[] toc::[] -You can enable link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2) in your cluster by editing the `node.config` object. Enabling cgroup v2 in {product-title} disables all cgroups version 1 controllers and hierarchies in your cluster. cgroup v1 is enabled by default. +By default, {product-title} uses link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v1.html[Linux control group version 1] (cgroup v1) in your cluster. You can switch to link:https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html[Linux control group version 2] (cgroup v2), if needed, by editing the `node.config` object. Enabling cgroup v2 in {product-title} disables all cgroup version 1 controllers and hierarchies in your cluster. cgroup v2 is the next version of the Linux cgroup API. cgroup v2 offers several improvements over cgroup v1, including a unified hierarchy, safer sub-tree delegation, new features such as link:https://www.kernel.org/doc/html/latest/accounting/psi.html[Pressure Stall Information], and enhanced resource management and isolation. -:FeatureName: {product-title} cgroups version 2 support -include::snippets/technology-preview.adoc[leveloffset=+0] - // The following include statements pull in the module files that comprise // the assembly. Include any combination of concept, procedure, or reference // modules required to cover the user story. You can also include other @@ -21,7 +18,7 @@ include::snippets/technology-preview.adoc[leveloffset=+0] include::modules/nodes-clusters-cgroups-2.adoc[leveloffset=+1] +[role="_additional-resources"] .Additional resources -* xref:../../nodes/clusters/nodes-cluster-enabling-features.adoc#nodes-cluster-enabling[Enabling OpenShift Container Platform features using FeatureGates] * xref:../../installing/index.adoc#ocp-installation-overview[OpenShift Container Platform installation overview] diff --git a/nodes/clusters/nodes-cluster-cgroups-okd.adoc b/nodes/clusters/nodes-cluster-cgroups-okd.adoc index 77b89bba43..b938792a38 100644 --- a/nodes/clusters/nodes-cluster-cgroups-okd.adoc +++ b/nodes/clusters/nodes-cluster-cgroups-okd.adoc @@ -1,7 +1,7 @@ :_content-type: ASSEMBLY :context: nodes-cluster-cgroups-2 [id="nodes-cluster-cgroups-okd"] -= Configuring the Linux cgroup on your nodes += Configuring the Linux cgroup version on your nodes include::_attributes/common-attributes.adoc[] toc::[] diff --git a/post_installation_configuration/cluster-tasks.adoc b/post_installation_configuration/cluster-tasks.adoc index 7119e12385..599bc5d4fb 100644 --- a/post_installation_configuration/cluster-tasks.adoc +++ b/post_installation_configuration/cluster-tasks.adoc @@ -630,10 +630,16 @@ include::modules/deploying-resource.adoc[leveloffset=+2] ifndef::openshift-origin[] include::modules/nodes-clusters-cgroups-2.adoc[leveloffset=+1] + +[role="_additional-resources"] +.Additional resources + +* xref:../nodes/clusters/nodes-cluster-cgroups-2.adoc#nodes-cluster-cgroups-2[Configuring the Linux cgroup version on your nodes] endif::openshift-origin[] ifdef::openshift-origin[] include::modules/nodes-clusters-cgroups-okd-configure.adoc[leveloffset=+1] endif::openshift-origin[] + [id="post-install-tp-tasks"] == Enabling Technology Preview features using FeatureGates