openstack: Test Nova AZ manifest generation

* instrument the manifest-generation test-framework to execute Python
testfiles
* test machinesets with availability zones

hack/openstack/test-manifests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -Eeuxo pipefail
+set -Eeuo pipefail
 
 [[ $(type -P yq) ]] || { >&2 echo "Required tool 'yq' not found in PATH" ; exit 1; }
 
@@ -12,6 +12,10 @@ declare \
 	external_network='' \
 	compute_flavor=''
 
+# Let install-config describe a configuration that is incompatible with the
+# target CI infrastructure
+export OPENSHFIT_INSTALL_SKIP_PREFLIGHT_VALIDATIONS=1
+
 print_help() {
 	set +x
 
@@ -47,7 +51,6 @@ fill_install_config() {
 }
 
 validate_configuration() {
-	set +x
 	declare -a required_values=("os_cloud" "external_network" "compute_flavor")
 	declare fail=false
 
@@ -63,7 +66,6 @@ validate_configuration() {
 		print_help
 		exit 1
 	fi
-	set -x
 }
 
 while getopts a:c:e:f:i:t:h o; do
@@ -92,27 +94,27 @@ validate_configuration
 
 >&2 echo "Running the tests from '${tests_dir}' against the Installer binary '${openshift_install}'."
 
-declare -i failed_tests=0
+declare result='PASS'
 for testcase in "${tests_dir}"/* ; do
 	if [ -d "$testcase" ]; then
 		assets_dir="$(mktemp -d)"
 		temp_dirs+=("$assets_dir")
 		fill_install_config "${testcase}/install-config.yaml" > "${assets_dir}/install-config.yaml"
 		"$openshift_install" create manifests --dir "$assets_dir"
-		for t in "${testcase}"/*.sh; do
+		for t in "${testcase}"/test_*; do
 			if $t "$assets_dir"; then
 				echo "PASS: '$t'"
 			else
-				declare -i f="$?"
-				echo "FAIL: $f failed tests in '$t'."
-				failed_tests=$((failed_tests+f))
+				result='FAIL'
+				echo "FAIL: '$t'"
 			fi
 		done
 	fi
 done
 
-if [ $failed_tests == 0 ]; then
-	echo 'PASS'
-else
-	echo "FAIL: ${failed_tests} failed tests."
+if [ "$result" != 'PASS' ]; then
+	echo "FAIL"
+	exit 1
 fi
+
+echo 'PASS'

images/openstack/Dockerfile.ci

@@ -41,7 +41,7 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 
 RUN mkdir /output && chown 1000:1000 /output
 USER 1000:1000
-ENV PATH /bin
+ENV PATH /bin:/usr/local/bin
 ENV HOME /output
 ENV LC_ALL en_US.UTF-8
 WORKDIR /output

scripts/openstack/manifest-tests/additional-security-group/test_masters.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 
-set -Eeuxo pipefail
+set -Eeuo pipefail
 
 declare -r assets_dir="$1"
+declare result='PASS'
 
 declare -a machines=(
 	"${assets_dir}/openshift/99_openshift-cluster-api_master-machines-0.yaml"
@@ -10,12 +11,10 @@ declare -a machines=(
 	"${assets_dir}/openshift/99_openshift-cluster-api_master-machines-2.yaml"
 )
 
-declare -i exit_code=0
-
 for machine in "${machines[@]}"; do
 	if ! [ -f "$machine" ]; then
 		>&2 echo "Machine resource $machine not found"
-		exit_code=$((exit_code+1))
+		result='FAIL'
 	fi
 
 	if ! >/dev/null yq -e '.spec.providerSpec.value.securityGroups[] | select(.uuid=="aaaaaaaa-bbbb-4ccc-dddd-111111111111")' "$machine"; then
@@ -24,8 +23,10 @@ for machine in "${machines[@]}"; do
 		>&2 echo 'The file was:'
 		>&2 cat "$machine"
 		>&2 echo
-		exit_code=$((exit_code+1))
+		result='FAIL'
 	fi
 done
 
-exit $exit_code
+if [ "$result" != 'PASS' ]; then
+	exit 1
+fi

scripts/openstack/manifest-tests/additional-security-group/test_workers.sh

@@ -1,16 +1,15 @@
 #!/usr/bin/env bash
 
-set -Eeuxo pipefail
+set -Eeuo pipefail
 
 declare -r assets_dir="$1"
+declare result='PASS'
 
 declare machineset="${assets_dir}/openshift/99_openshift-cluster-api_worker-machineset-0.yaml"
 
-declare -i exit_code=0
-
 if ! [ -f "$machineset" ]; then
 	>&2 echo 'MachineSet not found'
-	exit_code=$((exit_code+1))
+	result='FAIL'
 fi
 
 if ! >/dev/null yq -e '.spec.template.spec.providerSpec.value.securityGroups[] | select(.uuid=="aaaaaaaa-bbbb-4ccc-dddd-000000000000")' "$machineset"; then
@@ -19,7 +18,9 @@ if ! >/dev/null yq -e '.spec.template.spec.providerSpec.value.securityGroups[] |
 	>&2 echo 'The file was:'
 	>&2 cat "$machineset"
 	>&2 echo
-	exit_code=$((exit_code+1))
+	result='FAIL'
 fi
 
-exit $exit_code
+if [ "$result" != 'PASS' ]; then
+	exit 1
+fi

scripts/openstack/manifest-tests/nova-availability-zones/install-config.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+baseDomain: shiftstack.example.com
+clusterID: manifests1
+controlPlane:
+  hyperthreading: Enabled
+  architecture: amd64
+  name: master
+  platform:
+    openstack:
+      type: ${COMPUTE_FLAVOR}
+  replicas: 3
+compute:
+- name: worker
+  platform:
+    openstack:
+      type: ${COMPUTE_FLAVOR}
+      zones:
+      - zone
+      - ztwo
+      - zthree
+  replicas: 1000
+metadata:
+  name: manifests1
+networking:
+  clusterNetwork:
+  - cidr: 10.128.0.0/14
+    hostPrefix: 23
+  machineNetwork:
+  - cidr: 10.0.128.0/17
+  networkType: OpenShiftSDN
+  serviceNetwork:
+  - 172.30.0.0/16
+platform:
+  openstack:
+    cloud: ${OS_CLOUD}
+    externalNetwork: ${EXTERNAL_NETWORK}
+    computeFlavor: ${COMPUTE_FLAVOR}  # deprecated in 4.7
+    lbFloatingIP: ${API_FIP}
+pullSecret: ${PULL_SECRET}

scripts/openstack/manifest-tests/nova-availability-zones/test_machinesets.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import unittest
+
+import sys
+import glob
+import yaml
+
+ASSETS_DIR = ""
+
+EXPECTED_MACHINES_NUMBER = 1000
+EXPECTED_ZONE_NAMES = ["zone", "ztwo", "zthree"]
+
+
+class TestAZMachinesets(unittest.TestCase):
+    def setUp(self):
+        """Parse the MachineSets into a Python data structure."""
+        self.machinesets = []
+        for machineset_path in glob.glob(
+                f'{ASSETS_DIR}/openshift/99_openshift-cluster-api_worker-machineset-*.yaml'
+        ):
+            with open(machineset_path) as f:
+                self.machinesets.append(yaml.load(f, Loader=yaml.FullLoader))
+
+    def test_machineset_zone_name(self):
+        """Assert that there is exactly one MachineSet per availability zone."""
+        found = []
+        for machineset in self.machinesets:
+            zone = machineset["spec"]["template"]["spec"]["providerSpec"][
+                "value"]["availabilityZone"]
+            self.assertIn(zone, EXPECTED_ZONE_NAMES)
+            self.assertNotIn(zone, found)
+            found.append(zone)
+        self.assertEqual(len(self.machinesets), len(EXPECTED_ZONE_NAMES))
+
+    def test_total_replica_number(self):
+        """Assert that replicas spread across the MachineSets add up to the expected number."""
+        total_found = 0
+        for machineset in self.machinesets:
+            total_found += machineset["spec"]["replicas"]
+        self.assertEqual(total_found, EXPECTED_MACHINES_NUMBER)
+
+    def test_replica_distribution(self):
+        """Assert that replicas are evenly distributed across machinesets."""
+        setpoint = 0
+        for machineset in self.machinesets:
+            replicas = machineset["spec"]["replicas"]
+            if setpoint == 0:
+                setpoint = replicas
+            else:
+                self.assertTrue(-2 < replicas - setpoint < 2)
+
+
+if __name__ == '__main__':
+    ASSETS_DIR = sys.argv.pop()
+    unittest.main()