From d63debee34b9595239c4267dea162c43bed9e964 Mon Sep 17 00:00:00 2001 From: staebler Date: Fri, 12 Mar 2021 18:19:38 -0500 Subject: [PATCH] * changed "start" and "end" phases to "service start" and "service end" * add "pre-command start", "pre-command end", "post-command start" and "post-command end" phases * fixed issue where the kubelet was not notifying systemd that it had started since it had been moved to a script --- .../usr/local/bin/bootstrap-service-record.sh | 47 ++++++++++++++----- .../systemd/units/kubelet.service.template | 1 + docs/dev/bootstrap_services.md | 9 ++-- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/bootstrap-service-record.sh b/data/data/bootstrap/files/usr/local/bin/bootstrap-service-record.sh index a5db2d376f..3f5321628e 100644 --- a/data/data/bootstrap/files/usr/local/bin/bootstrap-service-record.sh +++ b/data/data/bootstrap/files/usr/local/bin/bootstrap-service-record.sh @@ -2,27 +2,34 @@ # This library provides a helper functions for recording when a service # and its stages start and end. +### +# When running as a pre- or post-command, set PRE_COMMAND or POST_COMMAND, respectively. +# These must be set *prior* to sourcing this script. +# PRE_COMMAND is the name identifying the pre-command being run. +# POST_COMMAND is the name identifying the post-command being run. + # SERVICE_RECORDS_DIR is the directory under which service records will be stored. SERVICE_RECORDS_DIR="${SERVICE_RECORDS_DIR:-/var/log/openshift/}" # SYSTEMD_UNIT_NAME is the name of the systemd unit for the service SYSTEMD_UNIT_NAME="$(ps -o unit= $$)" # SERVICE_NAME is the name of the service SERVICE_NAME="${SERVICE_NAME:-${SYSTEMD_UNIT_NAME%.service}}" -# STAGE_NAME is the name of the current stage in the service -STAGE_NAME="" - # add_service_record_entry adds a record entry to the service records file. -# PHASE - phase being recorded; one of "start", "end", "stage start", or "stage end" +# PHASE - phase being recorded; one of "service start", "service end", "stage start", "stage end", "pre-command start", +# "pre-command end", "post-command start", "post-command end" # RESULT - result of the action -# MESSAGE - message giving more detail about the result of the action +# STAGE (optional) - stage of the service +# PRE_COMMAND (optional) - name of the pre-command +# POST_COMMAND (optional) - name of the post-command +# ERROR_LINE (optional) - line where the error occurred +# ERROR_MESSAGE (optional) - message for the error add_service_record_entry() { local FILENAME="${SERVICE_RECORDS_DIR}/${SERVICE_NAME}.json" mkdir --parents $(dirname "${FILENAME}") # Append the new entry to the existing array in the file. # If the file does not already exist, start with an empty array. - # The new entry contains only the fields that have non-empty values, since the - # stage, result, error line, and error message are all optional. + # The new entry contains only the fields that have non-empty values, to omit optional values that were not provided. ([ -f "${FILENAME}" ] && cat "${FILENAME}" || echo '[]') | \ jq \ --arg timestamp "$(date +"%Y-%m-%dT%H:%M:%SZ")" \ @@ -43,7 +50,15 @@ add_service_record_entry() { # record_service_start() records the start of a service. record_service_start() { - local PHASE="start" + if [ "${PRE_COMMAND-}" ] + then + local PHASE="pre-command start" + elif [ "${POST_COMMAND-}" ] + then + local PHASE="post-command start" + else + local PHASE="service start" + fi add_service_record_entry } @@ -52,7 +67,15 @@ record_service_start() { # ERROR_LINE - line where the error occurred, if there was an error # ERROR_MESSAGE - error message, if there was an error record_service_end() { - local PHASE="end" + if [ "${PRE_COMMAND-}" ] + then + local PHASE="pre-command end" + elif [ "${POST_COMMAND-}" ] + then + local PHASE="post-command end" + else + local PHASE="service end" + fi local RESULT=${1:?Must specify a result} add_service_record_entry @@ -60,7 +83,7 @@ record_service_end() { # record_service_stage_start(stage_name) records the start of a stage of a service. record_service_stage_start() { - if [ -n "${STAGE_NAME}" ] + if [ "${STAGE_NAME-}" ] then echo "attempt to record the start of a stage without ending the previous one" exit 1 @@ -76,7 +99,7 @@ record_service_stage_start() { # ERROR_LINE - line where the error occurred, if there was an error # ERROR_MESSAGE - error message, if there was an error record_service_stage_end() { - if [ -z "${STAGE_NAME}" ] + if [ -z "${STAGE_NAME-}" ] then echo "attempt to record the end of a stage without starting one" exit 1 @@ -113,7 +136,7 @@ record_service_exit() { get_error_info ERROR_LINE ERROR_MESSAGE fi - if [ -n "${STAGE_NAME}" ] + if [ "${STAGE_NAME-}" ] then record_service_stage_end "${RESULT}" fi diff --git a/data/data/bootstrap/systemd/units/kubelet.service.template b/data/data/bootstrap/systemd/units/kubelet.service.template index 8d02035eff..092d4c8e6e 100644 --- a/data/data/bootstrap/systemd/units/kubelet.service.template +++ b/data/data/bootstrap/systemd/units/kubelet.service.template @@ -5,6 +5,7 @@ After=crio.service release-image.service [Service] Type=notify +NotifyAccess=all ExecStartPre=/bin/mkdir --parents /etc/kubernetes/manifests ExecStartPre=/bin/mkdir --parents /etc/kubernetes/kubelet-plugins/volume/exec ExecStartPre=/usr/local/bin/kubelet-pause-image.sh diff --git a/docs/dev/bootstrap_services.md b/docs/dev/bootstrap_services.md index f8960d902c..97759a9e80 100644 --- a/docs/dev/bootstrap_services.md +++ b/docs/dev/bootstrap_services.md @@ -12,7 +12,10 @@ the line number of the error and the last three lines from the service's journal * A service adds an entry when a stage of the service starts. An example of a service stage is the cvo-bootstrap stage of the bootkube service. During that stage, the cluster-version-operator renders its manifests. -* A service adds an entry when a stage of the service ends. This is similar to the entry +* A service adds an entry when a stage of the service ends. This is similar to the entry +added when a service ends, including having a result and error information if applicable. +* A service adds an entry when a pre- or post-command starts. +* A service adds en entry when a pre- or post-command ends. This is similar to the entry added when a service ends, including having a result and error information if applicable. ##### Managing Service Records in a Service ##### @@ -41,8 +44,8 @@ source the same /usr/local/bin/bootstrap-service-record.sh script. It should als `PRE_COMMAND` or `POST_COMMAND` environment variable with a value that identifies the command. For example, kubelet.service has a pre-command of /usr/local/bin/kubelet-pause-image.sh. The kubelet-pause-image.sh script sets the `PRE_COMMAND` environment variable to "kubelet-pause-image" -before sourcing the bootstrap-service-record.sh script. All of the entries for the pre-command -will contain a `preCommand` field with the "kubelet-pause-image" value. +before sourcing the bootstrap-service-record.sh script. The entries for the pre-command will +contain a `preCommand` field with the "kubelet-pause-image" value. ###### Sample Script #######