Double-check pod readiness for deployments (#10)

README.md

@@ -72,7 +72,7 @@ chart-test.sh
 ### Linting Charts
 
 ```shell
-docker run --rm -v "$(pwd):/workdir" --workdir /workdir gcr.io/kubernetes-charts-ci/chart-testing:v1.0.2 chart_test.sh --no-install --config .mytestenv
+docker run --rm -v "$(pwd):/workdir" --workdir /workdir gcr.io/kubernetes-charts-ci/chart-testing:v1.0.3 chart_test.sh --no-install --config .mytestenv
 ```
 
 *Sample Output*
@@ -131,12 +131,12 @@ Done.
 
 ### Installing and Testing Charts
 
-Installing a chart requires access to a Kubernetes cluster. You may have to create your own Docker image that extends from `gcr.io/kubernetes-charts-ci/chart-testing:v1.0.2` in order to install additional tools (e. g. `google-cloud-sdk` for GKE). You could run a container in the background, run the required steps to authenticatre and initialize the `kubectl` context before you, and eventually run `chart_test.sh`.
+Installing a chart requires access to a Kubernetes cluster. You may have to create your own Docker image that extends from `gcr.io/kubernetes-charts-ci/chart-testing:v1.0.3` in order to install additional tools (e. g. `google-cloud-sdk` for GKE). You could run a container in the background, run the required steps to authenticatre and initialize the `kubectl` context before you, and eventually run `chart_test.sh`.
 
 Charts are installed into newly created namespaces that will be deleted again afterwards. By default, they are named by the chart, which may not be a good idea, especially when multiple PR jobs could be running for the same chart. `chart_lib.sh` looks for an environment variable `BUILD_ID` and uses it to name the namespace. Make sure you set it based on the pull request number.
 
 ```shell
-docker run --rm -v "$(pwd):/workdir" --workdir /workdir gcr.io/kubernetes-charts-ci/chart-testing:v1.0.2 chart_test.sh --no-lint --config .mytestenv
+docker run --rm -v "$(pwd):/workdir" --workdir /workdir gcr.io/kubernetes-charts-ci/chart-testing:v1.0.3 chart_test.sh --no-lint --config .mytestenv
 ```
 
 #### GKE Example

build.sh

@@ -18,7 +18,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-readonly IMAGE_TAG=v1.0.2
+readonly IMAGE_TAG=v1.0.3
 readonly IMAGE_REPOSITORY="gcr.io/kubernetes-charts-ci/chart-testing"
 readonly SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
 

chart_test.sh

@@ -92,6 +92,13 @@ main() {
 
     pushd "$REPO_ROOT" > /dev/null
 
+    for dir in "${CHART_DIRS[@]}"; do
+        if [[ ! -d "$dir" ]]; then
+            chartlib::error "Configured charts directory '$dir' does not exist"
+            exit 1
+        fi
+    done
+
     local exit_code=0
 
     read -ra changed_dirs <<< "$(chartlib::detect_changed_directories)"

examples/gke/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gcr.io/kubernetes-charts-ci/chart-testing:v1.0.2
+FROM gcr.io/kubernetes-charts-ci/chart-testing:v1.0.3
 
 ENV PATH /google-cloud-sdk/bin:$PATH
 ARG CLOUD_SDK_VERSION=200.0.0

lib/chartlib.sh

@@ -289,11 +289,30 @@ chartlib::install_chart_with_single_config() {
             helm install "$chart_dir" --name "$release" --namespace "$namespace" --wait --timeout "$TIMEOUT"
         fi
 
+        local error=
+
         # For deployments --wait may not be sufficient because it looks at 'maxUnavailable' which is 0 by default.
-        for deployment in $(kubectl get deployment --namespace "$namespace" --output jsonpath='{.items[*].metadata.name}'); do
+        for deployment in $(kubectl get deployments --namespace "$namespace" --output jsonpath='{.items[*].metadata.name}'); do
             kubectl rollout status "deployment/$deployment" --namespace "$namespace"
+
+            # 'kubectl rollout status' does not return a non-zero exit code when rollouts fail.
+            # We, thus, need to double-check here.
+
+            local jsonpath='{.status.conditions[?(@.type=="Ready")].status}'
+
+            for pod in $(chartlib::get_pods_for_deployment "$deployment" "$namespace"); do
+                ready=$(kubectl get pod "$pod" --namespace "$namespace" --output jsonpath="$jsonpath")
+                if [[ "$ready" != "True" ]]; then
+                    chartlib::error "Pod '$pod' did not reach ready state!"
+                    error=true
+                fi
+            done
         done
 
+        if [[ -n "$error" ]]; then
+            return 1
+        fi
+
         echo "Testing chart '$chart_dir' in namespace '$namespace'..."
         helm test "$release" --cleanup --timeout "$TIMEOUT"
 
@@ -304,6 +323,25 @@ chartlib::install_chart_with_single_config() {
     fi
 }
 
+# Returns the pods that are governed by a deployment.
+# Args:
+#   $1 The name of the deployment
+#   $2 The namespace
+chartlib::get_pods_for_deployment() {
+    local deployment="${1?Deployment is required}"
+    local namespace="${2?Namespace is required}"
+
+    local jq_filter='.spec.selector.matchLabels | to_entries | .[] | "\(.key)=\(.value)"'
+
+    local selectors
+    mapfile -t selectors < <(kubectl get deployment "$deployment" --namespace "$namespace" --output=json | jq -r "$jq_filter")
+
+    local selector
+    selector=$(chartlib::join_by , "${selectors[@]}")
+
+    kubectl get pods --selector "$selector" --namespace "$namespace" --output jsonpath='{.items[*].metadata.name}'
+}
+
 # Lints a chart for all custom values files matching '*.values.yaml'
 # in the 'ci' subdirectory.
 # Args:
@@ -470,3 +508,13 @@ chartlib::delete_namespace() {
 chartlib::error() {
     printf '\e[31mERROR: %s\n\e[39m' "$1" >&2
 }
+
+# Joins strings by a delimiters
+# Args:
+#   $1 The delimiter
+#   $* Additional args to join by the delimiter
+chartlib::join_by() {
+    local IFS="$1"
+    shift
+    echo "$*"
+}