From 044918ae8defdfd23f239488d3d6f550e0edf647 Mon Sep 17 00:00:00 2001 From: Ashley Hardin Date: Tue, 25 May 2021 12:04:41 -0400 Subject: [PATCH] Added Editing kubelet log level verbosity topic --- _topic_map.yml | 2 + modules/gathering-kubelet-logs.adoc | 37 +++++++++++ modules/log-verbosity-descriptions.adoc | 32 +++++++++ ...odifying-kubelet-as-one-time-scenario.adoc | 65 +++++++++++++++++++ ...stent-kubelet-log-level-configuration.adoc | 35 ++++++++++ .../editing-kubelet-log-level-verbosity.adoc | 16 +++++ 6 files changed, 187 insertions(+) create mode 100644 modules/gathering-kubelet-logs.adoc create mode 100644 modules/log-verbosity-descriptions.adoc create mode 100644 modules/modifying-kubelet-as-one-time-scenario.adoc create mode 100644 modules/persistent-kubelet-log-level-configuration.adoc create mode 100644 rest_api/editing-kubelet-log-level-verbosity.adoc diff --git a/_topic_map.yml b/_topic_map.yml index 31ec0cf93c..299c1c9531 100644 --- a/_topic_map.yml +++ b/_topic_map.yml @@ -2032,6 +2032,8 @@ Name: API reference Dir: rest_api Distros: openshift-enterprise,openshift-origin Topics: +- Name: Editing kubelet log level verbosity and gathering logs + File: editing-kubelet-log-level-verbosity - Name: API list File: index - Name: Common object reference diff --git a/modules/gathering-kubelet-logs.adoc b/modules/gathering-kubelet-logs.adoc new file mode 100644 index 0000000000..45404d88bc --- /dev/null +++ b/modules/gathering-kubelet-logs.adoc @@ -0,0 +1,37 @@ +[id="gathering-kubelet-logs_{context}"] += Gathering kubelet logs + +.Procedure + +* After the kubelet's log level verbosity is configured properly, you can gather logs by running the following commands: ++ +[source,terminal] +---- +$ oc adm node-logs --role master -u kubelet +---- ++ +[source,terminal] +---- +$ oc adm node-logs --role worker -u kubelet +---- ++ +Alternatively, inside the node, run the following command: ++ +[source,terminal] +---- +$ journalctl -b -f -u kubelet.service +---- + +* To collect master container logs, run the following command: ++ +[source,terminal] +---- +$ sudo tail -f /var/log/containers/* +---- + +* To directly gather the logs of all nodes, run the following command: ++ +[source,terminal] +---- +- for n in $(oc get node --no-headers | awk '{print $1}'); do oc adm node-logs $n | gzip > $n.log.gz; done +---- diff --git a/modules/log-verbosity-descriptions.adoc b/modules/log-verbosity-descriptions.adoc new file mode 100644 index 0000000000..4c618949df --- /dev/null +++ b/modules/log-verbosity-descriptions.adoc @@ -0,0 +1,32 @@ +[id="log-verbosity-descriptions_{context}"] += Log verbosity descriptions + +[cols="2a,8a",options="header"] +|=== +|Log verbosity +|Description + +|`--v=0` +|Always visible to an Operator. + +|`--v=1` +|A reasonable default log level if you do not want verbosity. + +|`--v=2` +|Useful steady state information about the service and important log messages that might correlate to significant changes in the system. This is the recommended default log level. + +|`--v=3` +|Extended information about changes. + +|`--v=4` +|Debug level verbosity. + +|`--v=6` +|Display requested resources. + +|`--v=7` +|Display HTTP request headers. + +|`--v=8` +|Display HTTP request contents. +|=== diff --git a/modules/modifying-kubelet-as-one-time-scenario.adoc b/modules/modifying-kubelet-as-one-time-scenario.adoc new file mode 100644 index 0000000000..430d03425a --- /dev/null +++ b/modules/modifying-kubelet-as-one-time-scenario.adoc @@ -0,0 +1,65 @@ +[id="modifying-kubelet-one-time_{context}"] += Modifying the kubelet as a one-time scenario + +To modify the kubelet in a one-time scenario without rebooting the node due to the change of `machine-config(spec":{"paused":false}})`, allowing you to modify the kubelet without affecting the service, follow this procedure. + +.Procedure + +. Connect to the node in debug mode: ++ +[source,terminal] +---- +$ oc debug node/ +---- ++ +[source,terminal] +---- +$ chroot /host +---- + +. After access is established, check the content: ++ +[source,terminal] +---- +$ systemctl cat kubelet +---- ++ +.Example output +[source,terminal] +---- +# /etc/systemd/system/kubelet.service +mode: 0644 +path: "/etc/systemd/system/kubelet.service.d/20-logging.conf" +contents: + inline: | + [Service] + Environment="KUBELET_LOG_LEVEL=2" +---- + +. Define the new verbosity required in the `/etc/systemd/system/kubelet.service.d/20-logging.conf` file. In this example, the verbosity is changed from `v=1` to `v=8`: ++ +[source,terminal] +---- +$ vi -i -e 's/--v=1/--v=8/g' /etc/systemd/system/kubelet.service.d/20-logging.conf +---- ++ +Editing the config file or installing a new `logging.conf` file overrides the log level. + +. Restart the service: ++ +[source,terminal] +---- +$ systemctl daemon-reload +---- ++ +[source,terminal] +---- +$ systemctl restart kubelet +---- + +. Gather the logs, then edit the kubelet log level to revert to the former value to prevent issues, such as this error: ++ +[source,terminal] +---- +E0514 12:47:17.998892 2281 daemon.go:1350] content mismatch for file /etc/systemd/system/kubelet.service: [Unit] +---- diff --git a/modules/persistent-kubelet-log-level-configuration.adoc b/modules/persistent-kubelet-log-level-configuration.adoc new file mode 100644 index 0000000000..848a0bffce --- /dev/null +++ b/modules/persistent-kubelet-log-level-configuration.adoc @@ -0,0 +1,35 @@ +[id="persistent-kubelet-log-level-configuration_{context}"] += Persistent kubelet log level configuration + +.Procedure + +* Use the following `MachineConfig` object for persistent kubelet log level configuration: ++ +[source,yaml] +---- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig + metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 99-master-kubelet-loglevel + spec: + config: + ignition: + config: {} + security: + tls: {} + timeouts: {} + version: 2.2.0 + systemd: + units: + - name: kubelet.service + enable: true + dropins: + - name: 30-logging.conf + contents: | + [Service] + Environment="KUBELET_LOG_LEVEL=2" +---- ++ +Generally, it is recommended to apply `0-4` as debug-level logs and `5-8` as trace-level logs. diff --git a/rest_api/editing-kubelet-log-level-verbosity.adoc b/rest_api/editing-kubelet-log-level-verbosity.adoc new file mode 100644 index 0000000000..1abafaabd8 --- /dev/null +++ b/rest_api/editing-kubelet-log-level-verbosity.adoc @@ -0,0 +1,16 @@ +[id="editing-kubelet-log-level-verbosity"] += Editing kubelet log level verbosity and gathering logs +include::modules/common-attributes.adoc[] +:context: editing-kubelet-log-level-verbosity + +toc::[] + +To troubleshoot some issues with nodes, establish the kubelet's log level verbosity depending on the issue to be tracked. + +include::modules/modifying-kubelet-as-one-time-scenario.adoc[leveloffset=+1] + +include::modules/persistent-kubelet-log-level-configuration.adoc[leveloffset=+1] + +include::modules/log-verbosity-descriptions.adoc[leveloffset=+1] + +include::modules/gathering-kubelet-logs.adoc[leveloffset=+1]