diff --git a/Jenkinsfile b/Jenkinsfile index 4043a67187..6cbe8a16a2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,149 +14,118 @@ def creds = [ ] ] -pipeline { - agent { - label 'worker' - } +def quay_creds = [ + usernamePassword( + credentialsId: 'quay-robot', + passwordVariable: 'QUAY_ROBOT_SECRET', + usernameVariable: 'QUAY_ROBOT_USERNAME' + ) +] +def builder_image = 'quay.io/coreos/tectonic-builder:v1.12' + +pipeline { + agent none options { timeout(time:60, unit:'MINUTES') timestamps() buildDiscarder(logRotator(numToKeepStr:'20')) } - environment { - TECTONIC_INSTALLER_ROLE= 'tectonic-installer' - GO_PROJECT = '/go/src/github.com/coreos/tectonic-installer' - MAKEFLAGS = '-j4' - } - stages { - stage('TerraForm: Syntax Check') { - agent { - docker { - image 'quay.io/coreos/tectonic-builder:v1.12' - } + stage('Build & Test') { + environment { + GO_PROJECT = '/go/src/github.com/coreos/tectonic-installer' + MAKEFLAGS = '-j4' } steps { - sh """#!/bin/bash -ex - make structure-check - """ - } - } + node('worker && ec2') { + withDockerContainer(builder_image) { + checkout scm + sh """#!/bin/bash -ex + mkdir -p \$(dirname $GO_PROJECT) && ln -sf $WORKSPACE $GO_PROJECT - stage('Generate docs') { - agent { - docker { - image 'golang:1.8' + # TODO: Remove me. + go get github.com/segmentio/terraform-docs + go get github.com/s-urbaniak/terraform-examples + + cd $GO_PROJECT/ + make structure-check + + cd $GO_PROJECT/installer + make clean + make tools + make build + + make dirtycheck + make lint + make test + """ + stash name: 'installer', includes: 'installer/bin/linux/installer' + stash name: 'sanity', includes: 'installer/bin/sanity' + } } } - steps { - sh """#!/bin/bash -ex - - # Prevent fatal: You don't exist. Go away! git error - git config --global user.name 'jenkins tectonic installer' - git config --global user.email 'jenkins-tectonic-installer@coreos.com' - go get github.com/segmentio/terraform-docs - - make docs - git diff --exit-code - """ - } - } - - stage('Generate examples') { - agent { - docker { - image 'golang:1.8' - } - } - steps { - sh """#!/bin/bash -ex - - # Prevent fatal: You don't exist. Go away! git error - git config --global user.name 'jenkins tectonic installer' - git config --global user.email 'jenkins-tectonic-installer@coreos.com' - go get github.com/s-urbaniak/terraform-examples - - make examples - git diff --exit-code - """ - } - } - - stage('Installer: Build & Test') { - agent { - docker { - image 'quay.io/coreos/tectonic-builder:v1.12' - } - } - steps { - checkout scm - sh "mkdir -p \$(dirname $GO_PROJECT) && ln -sf $WORKSPACE $GO_PROJECT" - sh "go get github.com/golang/lint/golint" - sh """#!/bin/bash -ex - go version - cd $GO_PROJECT/installer - - make clean - make tools - make build - # make build ran yarn install. check if yarn.lock is modified - if git status --short | grep 'yarn.lock' > /dev/null - then - echo 'Someone forgot to commit yarn.lock!' - exit 1 - fi - make lint - make test - """ - stash name: 'installer', includes: 'installer/bin/linux/installer' - stash name: 'sanity', includes: 'installer/bin/sanity' - } } stage("Smoke Tests") { - agent { - docker { - image 'quay.io/coreos/tectonic-builder:v1.12' - } - } steps { parallel ( "TerraForm: AWS": { - unstash 'installer' - unstash 'sanity' - withCredentials(creds) { - timeout(30) { - sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws.tfvars' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh create vars/aws.tfvars' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh test vars/aws.tfvars' - } - timeout(10) { - sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh destroy vars/aws.tfvars' + node('worker && ec2') { + withCredentials(creds) { + withDockerContainer(builder_image) { + checkout scm + unstash 'installer' + unstash 'sanity' + timeout(30) { + sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws.tfvars' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh create vars/aws.tfvars' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh test vars/aws.tfvars' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh destroy vars/aws.tfvars' + } + } } } }, - "TerraForm: AWS-experimental": { - unstash 'installer' - unstash 'sanity' - withCredentials(creds) { - timeout(5) { - sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws-exp.tfvars' + "TerraForm: AWS (Experimental)": { + node('worker && ec2') { + withCredentials(creds) { + withDockerContainer(builder_image) { + checkout scm + unstash 'installer' + timeout(5) { + sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws-exp.tfvars' + } + } } } }, "TerraForm: AWS-custom-ca": { - unstash 'installer' - unstash 'sanity' - withCredentials(creds) { - timeout(5) { - sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws-ca.tfvars' + node('worker && ec2') { + withCredentials(creds) { + withDockerContainer(builder_image) { + checkout scm + unstash 'installer' + timeout(5) { + sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh plan vars/aws-ca.tfvars' + } + } + } + } + }, + "Terraform: Bare Metal": { + node('worker && bare-metal') { + checkout scm + unstash 'installer' + unstash 'sanity' + withCredentials(creds) { + timeout(30) { + sh '${WORKSPACE}/tests/smoke/bare-metal/smoke.sh vars/metal.tfvars' + } } } } @@ -164,11 +133,18 @@ pipeline { } post { failure { - unstash 'installer' - withCredentials(creds) { - timeout(10) { - sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' - sh '${WORKSPACE}/tests/smoke/aws/smoke.sh destroy vars/aws.tfvars' + node('worker && ec2') { + withCredentials(creds) { + withDockerContainer(builder_image) { + checkout scm + unstash 'installer' + sh 'set +x -e && eval "$(${WORKSPACE}/tests/smoke/aws/smoke.sh assume-role "$TECTONIC_INSTALLER_ROLE")"' + retry(3) { + timeout(15) { + sh '${WORKSPACE}/tests/smoke/aws/smoke.sh destroy vars/aws.tfvars' + } + } + } } } } @@ -180,30 +156,19 @@ pipeline { branch 'master' } steps { - unstash 'installer' - unstash 'sanity' - withCredentials([ - usernamePassword( - credentialsId: 'quay-robot', - passwordVariable: 'QUAY_ROBOT_SECRET', - usernameVariable: 'QUAY_ROBOT_USERNAME' - ) - ]) { - sh """ - docker build -t quay.io/coreos/tectonic-installer:master -f images/tectonic-installer/Dockerfile . - docker login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_SECRET" quay.io - docker push quay.io/coreos/tectonic-installer:master - docker logout quay.io - """ + node('worker && ec2') { + withCredentials(quay_creds) { + checkout scm + unstash 'installer' + sh """ + docker build -t quay.io/coreos/tectonic-installer:master -f images/tectonic-installer/Dockerfile . + docker login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_SECRET" quay.io + docker push quay.io/coreos/tectonic-installer:master + docker logout quay.io + """ + } } } } } - - post { - always { - // Cleanup workspace - deleteDir() - } - } } diff --git a/Makefile b/Makefile index b842c0d0fb..fa46d606bc 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ structure-check: $(eval FMT_ERR := $(shell terraform fmt -list -write=false .)) @if [ "$(FMT_ERR)" != "" ]; then echo "misformatted files (run 'terraform fmt .' to fix):" $(FMT_ERR); exit 1; fi -canonical-syntax: - terraform fmt -list . + @if make docs && ! git diff --exit-code; then echo "outdated docs (run 'make docs' to fix)"; exit 1; fi + @if make examples && ! git diff --exit-code; then echo "outdated examples (run 'make examples' to fix)"; exit 1; fi -.PHONY: make clean terraform terraform-dev structure-check canonical-syntax docs examples terraform-get +.PHONY: make clean terraform terraform-dev structure-check docs examples terraform-get diff --git a/installer/Makefile b/installer/Makefile index 252aa1f2a6..f222e2b32e 100644 --- a/installer/Makefile +++ b/installer/Makefile @@ -62,13 +62,9 @@ test: test-backend test-frontend shellcheck: $(SHELL_FILES) shellcheck $(SHELL_FILES) -.PHONY: smoke-aws -smoke-aws: bin/sanity - ../tests/scripts/aws/up-down.sh - -.PHONY: smoke-bare-metal -smoke-bare-metal: bin/sanity - ./tests/scripts/bare-metal/up-down.sh +.PHONY: dirtycheck +dirtycheck: + @if git status --short | grep 'yarn.lock' > /dev/null; then echo "outdated yarn.lock (commit it to fix)"; exit 1; fi .PHONY: dist dist: release-bins diff --git a/installer/tests/scripts/aws/README.md b/installer/tests/scripts/aws/README.md deleted file mode 100644 index 25fd4a2d99..0000000000 --- a/installer/tests/scripts/aws/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# Installer Test Scripts - -These tools automate using the installer and provide simple tools to work with the cluster once its launched. - -## Basic Usage -These commands bring up a cluster, wait for it to be ready, setup and start bootkube, and check for cluster availability. - -```bash -# set AWS credential environment variables -export AWS_ACCESS_KEY_ID=[actual key] -export AWS_SECRET_ACCESS_KEY=[actual key] - -# set cluster environment variables for new environment -source ./default.env.sh - -# use installer to provision cluster -./launch_cluster_aws.sh - -# wait until cluster is available -./wait_for_dns.sh - -# transfer configuration and start bootkube -./setup_bootkube.sh -sleep 5 - -# check for API server availability and Pod status -kubectl --kubeconfig=./output/${CLUSTER_NAME}/assets/auth/kubeconfig get pods --all-namespaces -kubectl --kubeconfig=./output/${CLUSTER_NAME}/assets/auth/kubeconfig get nodes -``` - -## Configuration -**aws_payload.tmpl.sh** -The JSON body of the request to the Tectonic installer - -**default.env.sh** -The default values for new clusters, options are documented there. - -### Console auth -Username: `admin@example.com` -Password: `tectonicTestPass11042016` - -### Switching clusters -If you are working with multiple clusters and want to switch the one you are working on, each cluster profile has an 'env' file with the correct environment variables to use that profile. To use it set the variables in your shell by running: -```bash -source ./output/[Cluster Name]/env -``` -Replace "Cluster Name" with the name of your cluster. diff --git a/installer/tests/scripts/aws/aws_payload.tmpl.sh b/installer/tests/scripts/aws/aws_payload.tmpl.sh deleted file mode 100755 index bb2cf18c74..0000000000 --- a/installer/tests/scripts/aws/aws_payload.tmpl.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -cat < ${env_file} -export CLUSTER_NAME=${CLUSTER_NAME} -export AWS_REGION=${AWS_REGION} -export AWS_HOSTEDZONE=${AWS_HOSTEDZONE} -export CLUSTER_DOMAIN=${CLUSTER_DOMAIN} -export TECTONIC_DOMAIN=${TECTONIC_DOMAIN} -export UPDATER_ENABLED=${UPDATER_ENABLED} -export UPDATER_SERVER=${UPDATER_SERVER} -export UPDATER_CHANNEL=${UPDATER_CHANNEL} -export UPDATER_APPID=${UPDATER_APPID} -export TECTONIC_LICENSE=${TECTONIC_LICENSE} -export TECTONIC_PULL_SECRET=$(echo ${TECTONIC_PULL_SECRET} | sed 's/\\/\\\\/g') -EOF - -echo "A cluster configuration has been written to ${CLUSTER_DIR}." -echo "In order to use this cluster, you may need to run 'source ${env_file}'" - -echo "Starting Tectonic Installer" -${INSTALLER_BIN} -log-level=debug -open-browser=false -platforms=aws & -INSTALLER_PID=$! -sleep 2 - -echo "Submitting to Tectonic Installer" -assets_zip="${CLUSTER_DIR}/assets.zip" -echo "${PAYLOAD}" | >${assets_zip} curl -v -X POST --data-binary @- ${CLUSTER_CREATE_URL} -echo "Provisioning requested" - -# Killing quickly because multiple tests are using this port -echo "Killing Tectonic Installer" -kill ${INSTALLER_PID} || true - -echo "Unzipping self-hosting provisioning assets" -unzip ${assets_zip} -d ${CLUSTER_DIR} > /dev/null - diff --git a/installer/tests/scripts/aws/setup_bootkube.sh b/installer/tests/scripts/aws/setup_bootkube.sh deleted file mode 100755 index 3ba050fdb4..0000000000 --- a/installer/tests/scripts/aws/setup_bootkube.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -e -# This script transfers bootkube assets to a cluster and starts it. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT="$DIR/../../.." - -CLUSTER_DIR=${CLUSTER_DIR:-"${ROOT}/tests/scripts/aws/output/${CLUSTER_NAME}"} -assets_dir="${CLUSTER_DIR}/assets" - -no_hostcheck="-o stricthostkeychecking=no" -ssh_args="${no_hostcheck}" -target="core@${CLUSTER_DOMAIN}" - -until $(curl --silent --fail -m 1 "http://${CLUSTER_DOMAIN}:10255/healthz" > /dev/null); do - echo "Waiting for Kubelets to start..." - sleep 15 -done - -echo "Starting bootkube..." -ssh ${ssh_args} ${target} -- 'sudo systemctl start bootkube' - -until $(curl --silent --fail -k https://${TECTONIC_DOMAIN} > /dev/null); do - echo "Waiting for Tectonic Console..." - sleep 10 -done - -ssh ${ssh_args} ${target} -- 'sudo journalctl -u bootkube' diff --git a/installer/tests/scripts/aws/shutdown_aws.sh b/installer/tests/scripts/aws/shutdown_aws.sh deleted file mode 100755 index b94f56c083..0000000000 --- a/installer/tests/scripts/aws/shutdown_aws.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e -# This script destroys the current cluster. -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT="$DIR/../../.." - -source "${ROOT}/scripts/awsutil.sh" - -# require AWS creds -check_aws_creds - -RKT_STAGE1_IMG=${RKT_STAGE1_IMG:-"coreos.com/rkt/stage1-fly:1.17.0"} -RKT_OPTS="--interactive --stage1-name=${RKT_STAGE1_IMG} --insecure-options=image" -sudo rkt run ${RKT_OPTS} --set-env=AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} --set-env=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} docker://python:2.7.12-slim --exec /bin/bash < ${ROOT}/${logname} - aws_upload_file ${ROOT}/${logname} ${logname} ${bucketname} - rm "${ROOT}/${logname}" - - echo "${1} log uploaded at https://s3-us-west-2.amazonaws.com/tectonic-upgrade-test-logs/${logname}" -} - -function cleanup_coreupdate() { - echo "Deleting package ${VERSION}" - - ${UPDATESERVICECTL} --server ${SERVER} \ - --key ${COREUPDATE_KEY} \ - --user ${COREUPDATE_USR} \ - package delete \ - --app-id ${APPID} \ - --version ${VERSION} - - echo "Deleting channel ${CHANNEL}" - ${UPDATESERVICECTL} --server ${SERVER} \ - --key ${COREUPDATE_KEY} \ - --user ${COREUPDATE_USR} \ - channel delete \ - --app-id ${APPID} \ - --channel ${CHANNEL} - - echo "Deleting group ${CHANNEL}" - ${UPDATESERVICECTL} --server ${SERVER} \ - --key ${COREUPDATE_KEY} \ - --user ${COREUPDATE_USR} \ - group delete \ - --app-id ${APPID} \ - --group-id ${CHANNEL} - echo "Coreupdate server cleaned up" -} - -function finish() { - echo "Finishing the test" - - trap shutdown EXIT - - cleanup_coreupdate - upload_logs tectonic-channel-operator - upload_logs kube-version-operator - - shutdown -} - -function upload_payloads() { - echo "Uploading payload, assuming the payload (update-payload/payload.json) is up-to-date" - - ${ROOT}/update-payload/upload-payload.sh - - echo "Creating channel ${CHANNEL}" - ${UPDATESERVICECTL} --server ${SERVER} \ - --key ${COREUPDATE_KEY} \ - --user ${COREUPDATE_USR} \ - channel create \ - --app-id ${APPID} \ - --channel ${CHANNEL} \ - --version ${VERSION} - - echo "Creating group ${CHANNEL}" - ${UPDATESERVICECTL} --server ${SERVER} \ - --key ${COREUPDATE_KEY} \ - --user ${COREUPDATE_USR} \ - group create \ - --app-id ${APPID} \ - --channel ${CHANNEL} \ - --group-id ${CHANNEL} - - echo "Publishing payload to channel ${CHANNEL}" - ${ROOT}/update-payload/publish-payload.sh ${CHANNEL} - echo "Payload successfully published to channel ${CHANNEL}" - - trap finish EXIT -} - -function bringup_cluster() { - echo "Downloading kubectl" - - ${ROOT}/installer/scripts/bare-metal/get-kubectl $(pwd) - curl -O https://storage.googleapis.com/kubernetes-release/release/v1.5.2/bin/linux/amd64/kubectl && chmod +x ${ROOT}/kubectl - export KUBECTL=$(pwd)/kubectl - - echo "Fetching the release tarball" - - INSTALLER_DIR=${ROOT}/bin - URL="https://releases.tectonic.com/tectonic-${ORIGINAL_VERSION}.tar.gz" - curl -L -O ${URL} - mkdir -p ${INSTALLER_DIR} - tar -C ${INSTALLER_DIR} --strip-components=4 -xzf tectonic-${ORIGINAL_VERSION}.tar.gz ./tectonic/tectonic-installer/linux/installer - export INSTALLER_BIN=${INSTALLER_DIR}/installer - chmod +x ${INSTALLER_BIN} - - echo "Launching cluster" - - result=$(git checkout -b current) - if [[ $? != 0 ]]; then - git branch -D current - git checkout -b current - fi - git checkout tags/${ORIGINAL_VERSION} - - # ---- Start of up.sh ---- - - echo "setup environment" - source "${ROOT}/tests/scripts/aws/default.env.sh" - - echo "Publishing payload to staging channel" - ${ROOT}/scripts/update-payload/publish-payload.sh "tectonic-1.6-staging" - echo "Payload successfully published to tectonic-1.6-staging channel" - - echo "wait for machines to come up" - sleep 200 - ${ROOT}/tests/scripts/aws/wait_for_dns.sh - - echo "transfer config and start bootkube" - ${ROOT}/tests/scripts/aws/setup_bootkube.sh - - echo "Bootkube running!" - - # ---- End of up.sh ---- - - source ${ROOT}/scripts/awsutil.sh - - git checkout current - - echo "Configure kubectl" - CLUSTER_DIR=${CLUSTER_DIR:-"${ROOT}/tests/scripts/aws/output/${CLUSTER_NAME}"} - export KUBECONFIG="${CLUSTER_DIR}/assets/auth/kubeconfig" - - echo "Kubeconfig:" - cat ${KUBECONFIG} - - echo "Printing debug information:" - ${KUBECTL} get pods --all-namespaces - ${KUBECTL} get nodes -} - -# main: -# The upgrade test will be triggered by commenting "test upgrade from xxx", -# where "xxx" is the start version, e.g. "1.5.2-tectonic.2". -# The jenkins job can be found at https://jenkins-tectonic.prod.coreos.systems/job/tectonic-upgrade-test/ - -# ${ghprbTriggerCommentBody} should be in the form of: -# "test upgrade from 1.5.2-tectonic.1 -export DOCKER_CONFIG=$(dirname ${DOCKER_CONFIG_FILE}) -export ROOT=$(git rev-parse --show-toplevel) -export CLUSTER_NAME=${CLUSTER_NAME:-"upgrade-test-${BUILD_NUMBER}"} - -arr=(${ghprbCommentBody}) -payload=${ROOT}/update-payload/payload.json -export ORIGINAL_VERSION=${ORIGINAL_VERSION:-${arr[3]}} -export TARGET_VERSION=$(cat ${payload} | jq -r .version) -export KUBERNETES_TARGET_VERSION=$(cat ${payload} | jq -r '.desiredVersions[] | select(.name | contains("kubernetes")) |.version') - -export AWS_REGION="us-west-2" -export AWS_DEFAULT_REGION=${AWS_REGION} -export SERVER="https://public-update.staging.core-os.net" -export APPID="1f650cdd-878d-4550-9013-6786be951696" -export CHANNEL="upgrade-test-${BUILD_NUMBER}" -# The "version" on the coreupdate channel needs to be big enough -# so the TCO can receive it with the ${ORIGINAL_VERSION}. -export VERSION="${TARGET_VERSION}${BUILD_NUMBER}" - -export BUCKET="tectonic-update-payload-testing" -export DESTINATION="${CLUSTER_NAME}.json" - -export UPDATESERVICECTL="docker run -v ${ROOT}:${ROOT} quay.io/coreos/updateservicectl@sha256:471ef9637c16df765c4fd928e4c93f011e471bae7e4191022cc6a074f9b4b628" - -echo "Updating from ${ORIGINAL_VERSION} to ${TARGET_VERSION}" - -upload_payloads - -bringup_cluster - -sudo rkt run \ - --volume bk,kind=host,source=${ROOT} \ - --mount volume=bk,target=/go/src/github.com/coreos/tectonic-installer \ - --insecure-options=image docker://golang:1.7.5 \ - --user=$(id -u) \ - --set-env=KUBECONFIG_PATH="${ROOT}/tests/scripts/aws/output/${CLUSTER_NAME}/assets/auth/kubeconfig" \ - --set-env=ORIGINAL_VERSION=${ORIGINAL_VERSION} \ - --set-env=CHANNEL=${CHANNEL} \ - --set-env=TECTONIC_TARGET_VERSION=${TARGET_VERSION} \ - --set-env=KUBERNETES_TARGET_VERSION=${KUBERNETES_TARGET_VERSION} \ - --exec /bin/bash -- -c \ - "cd /go/src/github.com/coreos/tectonic-installer && go test -timeout 30m -v ./installer/tests/upgrade" diff --git a/installer/tests/scripts/aws/wait_for_dns.sh b/installer/tests/scripts/aws/wait_for_dns.sh deleted file mode 100755 index ed796a2dfe..0000000000 --- a/installer/tests/scripts/aws/wait_for_dns.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -# check if cluster domain is set -if [ -z "${CLUSTER_DOMAIN}" ]; then - echo "\$CLUSTER_DOMAIN must be set to check for DNS." - exit 1 -fi - -# TODO: Replace with check on cluster/status -echo "Checking if records exist for ${CLUSTER_DOMAIN}" -for i in `seq 1 50`; do - # hack to speed up NCACHE invalidation - dig @8.8.8.8 -t NS +short ${CLUSTER_DOMAIN#*.} >/dev/null || true - - # get IPs for cluster domain - ips=$(dig @8.8.8.8 -t A +short ${CLUSTER_DOMAIN}) - if [ "$(printf "${ips}" | wc -l)" != "0" ]; then - echo "Found records!" - printf "IPs:\n${ips}" - echo - break - fi - echo "no records found yet" - sleep 10 -done diff --git a/installer/tests/scripts/bare-metal/get-kubectl.sh b/installer/tests/scripts/bare-metal/get-kubectl.sh deleted file mode 100755 index b3c4f70333..0000000000 --- a/installer/tests/scripts/bare-metal/get-kubectl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# USAGE: ./get-kubectl bin -# Get the kubectl client -set -eu - -DEST=${1:-"bin"} -VERSION="v1.5.6" - -URL="https://storage.googleapis.com/kubernetes-release/release/${VERSION}/bin/linux/amd64/kubectl" - -if [[ ! -f $DEST/kubectl ]]; then - mkdir -p ${DEST} - curl -L -o ${DEST}/kubectl ${URL} - chmod +x ${DEST}/kubectl -fi diff --git a/installer/tests/scripts/bare-metal/up-down.sh b/installer/tests/scripts/bare-metal/up-down.sh deleted file mode 100755 index 129ffc5c48..0000000000 --- a/installer/tests/scripts/bare-metal/up-down.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -set -e pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOT="$DIR/../../.." - -export VM_MEMORY='2048' -export ASSETS_DIR="${ASSETS_DIR:-$GOPATH/src/github.com/coreos/matchbox/examples/assets}" -MATCHBOX_SHA=9a3347f1b5046c231f089374b63defb800b04079 -INSTALLER_BIN=${INSTALLER_BIN:-"$ROOT/bin/linux/installer"} -SANITY_BIN=${SANITY_BIN:="$ROOT/bin/sanity"} -CLUSTER_CREATE="http://127.0.0.1:4444/cluster/create" - -main() { - if [ -z "$TECTONIC_LICENSE" ] || [ -z "$TECTONIC_PULL_SECRET" ];then - echo "Must export both \$TECTONIC_LICENSE and \$TECTONIC_PULL_SECRET" - return 1 - fi - - TEMP=$(mktemp -d) - echo "Creating $TEMP" - - echo "Getting matchbox" - rm -rf matchbox - git clone https://github.com/coreos/matchbox - pushd matchbox - git checkout $MATCHBOX_SHA - chmod 600 tests/smoke/fake_rsa - popd - cp examples/fake-creds/{ca.crt,server.crt,server.key} matchbox/examples/etc/matchbox - - setup - trap cleanup EXIT - - echo "Starting matchbox" - pushd matchbox - sudo -S -E ./scripts/devnet create - popd - - echo "Starting Tectonic Installer" - ${INSTALLER_BIN} -log-level=debug -open-browser=false & INSTALLER_PID=$! - sleep 2 - - echo "Writing configuration" - cp ${ROOT}/examples/metal.json ${TEMP}/metal.json - sed -i "s//${TECTONIC_LICENSE}/" ${TEMP}/metal.json - sed -i "s//$(echo ${TECTONIC_PULL_SECRET} | sed 's/\\/\\\\/g')/" ${TEMP}/metal.json - - echo "Submitting to Tectonic Installer" - curl -H "Content-Type: application/json" -X POST -d @${TEMP}/metal.json ${CLUSTER_CREATE} -o $TEMP/assets.zip - unzip $TEMP/assets.zip -d $TEMP - - echo "Starting QEMU/KVM nodes" - pushd matchbox - sudo -E ./scripts/libvirt create - popd - - until kubelet "node1.example.com" \ - && kubelet "node2.example.com" \ - && kubelet "node3.example.com" - do - sleep 15 - echo "Waiting for Kubelets to start..." - done - - ssh core@node1.example.com 'sudo systemctl start bootkube' - - until [[ "$(readyNodes)" == "3" ]]; do - sleep 5 - echo "$(readyNodes) of 3 nodes are Ready..." - done - - echo "Getting nodes..." - k8s get nodes - - sleep 5 - until [[ "$(readyPods)" == "$(podCount)" && "$(readyPods)" -gt "0" ]]; do - sleep 15 - echo "$(readyPods) pods are Running..." - k8s get pods --all-namespaces || true - done - k8s get pods --all-namespaces || true - - until $(curl --silent --fail -k https://tectonic.example.com > /dev/null); do - echo "Waiting for Tectonic Console..." - k8s get pods --all-namespaces || true - sleep 15 - done - - export NODE_COUNT=3 - export TEST_KUBECONFIG="${TEMP}/assets/auth/kubeconfig" - echo "Running Go sanity tests" - ${SANITY_BIN} - - echo "Tectonic bare-metal cluster came up!" - echo - - echo "Cleaning up" - cleanup -} - -setup() { - ${DIR}/get-kubectl.sh - - pushd matchbox - sudo ./scripts/libvirt destroy || true - sudo ./scripts/devnet destroy || true - popd - sudo rkt gc --grace-period=0 -} - -kubelet() { - curl --silent --fail -m 1 http://$1:10255/healthz > /dev/null -} - -ssh() { - command ssh -i matchbox/tests/smoke/fake_rsa -o stricthostkeychecking=no "$@" -} - -k8s() { - ${ROOT}/bin/kubectl --kubeconfig=${TEMP}/assets/auth/kubeconfig "$@" -} - -# ready nodes returns the number of Ready Kubernetes nodes -readyNodes() { - k8s get nodes -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l -} - -# ready pods returns the number of Running pods -readyPods() { - k8s get pods --all-namespaces -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l -} - -# podCount returns the number of pods -podCount() { - k8s get pods --all-namespaces -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E status | wc -l -} - -cleanup() { - echo "Killing Tectonic Installer" - kill ${INSTALLER_PID} || true - - echo "Cleanup matchbox and VMs" - pushd matchbox - sudo ./scripts/libvirt destroy || true - sudo ./scripts/devnet destroy || true - popd - sudo rkt gc --grace-period=0 - rm -rf ${TEMP} -} - -main "$@" diff --git a/tests/smoke/bare-metal/fake-creds/ca.crt b/tests/smoke/bare-metal/fake-creds/ca.crt new file mode 100644 index 0000000000..3d69f45f56 --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/ca.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFDTCCAvWgAwIBAgIJAIuXq10k2OFlMA0GCSqGSIb3DQEBCwUAMBIxEDAOBgNV +BAMMB2Zha2UtY2EwHhcNMTcwMjAxMjIxMzI0WhcNMjcwMTMwMjIxMzI0WjASMRAw +DgYDVQQDDAdmYWtlLWNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +zzHsB56F6oZjsVBKzfpicsG+mVHQ/QzA4jqRCbQ8Zr12NtUZKnPUVwDoFf4WTfmy +Z0u8Uv+6/B/8un3LGsIaJEugPfRboc2oZKJcqfMJSFfLb/wkmT0D/1HJR60ml/M5 +wpHeh4vQ7BhktNsK90EjdlLvr1GDfevXArnye5ksEInOSX9nXVsGPrm0AGSffhmY +uUAjY8f9IspJa1j4vL6NI89GWO4jqME+SUnuI4SYIkuQJoSElofAIX2b5Tk3dFya +VKmAq2L89teCMYsciPbFa/Z2HvDNZ7pC17Ow7zr1f+V5BU18h3cLk610YNPcEBw0 +f94+mePsmMSMjUM0f+NMFyDERF+pys60/3qqVWrJe/FkJM6NDCyWXXXAfTxIwLq0 +CVrlWALdTc+RMAPI2sxAdUp4BqAuek4SjIg3FuoJrBs3EAUPfybclJ7g3HJwyXM2 +3WIe10BnSk+rGzd4KMVbYw5/nM8Nc/Y20R2an/vVZn6xTxs9o6hhEHF7d5iws6Bi +7/jv+jdZhLG8b3sG6Tj7a7YdvKWqH/mSPFlc/sevYOjR7NKYRMwGnl0d9qf+Xe5V +xyH1llIXPs6+y1B4tRyL/tulyeVqi25+I4QVAYypxWU8CPyw7tsSdOsSTbeGTmXj +ehelY/BCjAqAcexL7oRV7dy7VZ1Ezg6zQRwMt0Tar90CAwEAAaNmMGQwHQYDVR0O +BBYEFNGPoXTjJnHjG2zMpjSg/9vNO/trMB8GA1UdIwQYMBaAFNGPoXTjJnHjG2zM +pjSg/9vNO/trMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMA0G +CSqGSIb3DQEBCwUAA4ICAQC9V/0iiEZYHz7xbezHpeGHwmecH5oylEvAeCcN10gx +HFvUN+XMyBaPqN7iRtx/rSqyp2iN2AK1Cdn1viOSRc09lwPiuj9V4diSDyPwJWxd +60gqd5E9F9gQXlenWoIdm7kW8Lo8HLfx8ItYKGpE51JUctTmGY5WURRmBlVKr1LA +hbVsAWBaGQfPyW1CrFcxxc5mCABxWOxjRjLw8A8c5IXD0Q5C5pRd0BckBHKTdl40 +owm893oPEQcu/1C432T4vIddVh1Ktq1pd7O/9BPYOaPryzf7076xSwZ0bSuBUGRq +Vd3STfu5QRqpMv4dIrhqRofmIUzjOHLRX8Lx2pzgYcMgMQ8O+jM+ETrYD6rsDoLQ +uiVSWZK0YFndKzNTA04u57arRumWKqqfS0kkDFayumyv6KaDS6YZdsqSRmaiLAOG +F6jchpUtkDhDY0v/Y7jESUneT0hRnqNMPAKJMNhE4hS+1qkcP/ikQQgZl/OWma1z +HUyBGT4OGP2T3JIfq12Z4vC5FGVD4aD/frTvPMlifV3i8lKlYZs271JPXUo6ASIA +ZSBpV5QilOlE25Q5Lcw0yWmN4KwxqBL9bJ5W9D1I0qhWxaMF78m+8vLIFv+dAylE +Od27a+1We/P5ey7WRlwCfuEcFV7nYS/qMykYdQ9fxHSPgTPlrGrSwKstaaIIqOkE +kA== +-----END CERTIFICATE----- diff --git a/tests/smoke/bare-metal/fake-creds/ca.key b/tests/smoke/bare-metal/fake-creds/ca.key new file mode 100644 index 0000000000..9a81a1d2bd --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAzzHsB56F6oZjsVBKzfpicsG+mVHQ/QzA4jqRCbQ8Zr12NtUZ +KnPUVwDoFf4WTfmyZ0u8Uv+6/B/8un3LGsIaJEugPfRboc2oZKJcqfMJSFfLb/wk +mT0D/1HJR60ml/M5wpHeh4vQ7BhktNsK90EjdlLvr1GDfevXArnye5ksEInOSX9n +XVsGPrm0AGSffhmYuUAjY8f9IspJa1j4vL6NI89GWO4jqME+SUnuI4SYIkuQJoSE +lofAIX2b5Tk3dFyaVKmAq2L89teCMYsciPbFa/Z2HvDNZ7pC17Ow7zr1f+V5BU18 +h3cLk610YNPcEBw0f94+mePsmMSMjUM0f+NMFyDERF+pys60/3qqVWrJe/FkJM6N +DCyWXXXAfTxIwLq0CVrlWALdTc+RMAPI2sxAdUp4BqAuek4SjIg3FuoJrBs3EAUP +fybclJ7g3HJwyXM23WIe10BnSk+rGzd4KMVbYw5/nM8Nc/Y20R2an/vVZn6xTxs9 +o6hhEHF7d5iws6Bi7/jv+jdZhLG8b3sG6Tj7a7YdvKWqH/mSPFlc/sevYOjR7NKY +RMwGnl0d9qf+Xe5VxyH1llIXPs6+y1B4tRyL/tulyeVqi25+I4QVAYypxWU8CPyw +7tsSdOsSTbeGTmXjehelY/BCjAqAcexL7oRV7dy7VZ1Ezg6zQRwMt0Tar90CAwEA +AQKCAgEAjH2XQ9tThqC1fIerEVvT4WhJ6wA1K0C4kS2RJvlVc3zIaYm5VLXRp2Tv ++emeCiVjuPL7sXPBwC+YWIPvcidnPnEhKKFGeMJQilwlZP9srecKBNb9ogJjcX5t +cvKPlrzPz4TFVTeS5GPt9UwJdXpvp025RDGLbZi65BhduT01ScmHXQLMfdq4s1OM +IDAajZChpAs/c+spU6vCeM2Na73xSfTECI0BFO5jY6KDnQXNeoOuLM/yb3eA6bSY +Pqe7WGVqKDn/CzdFu8KJfzqKkLxzRS+LDJPPU6RSqpwnPy/FQ4G/u768z8YCzZHx +ta4yK6JUXte9ru+DgFrVyvtk38qpzlNYj5PVPkZxOZaWPALYAa6N53/NSJIZ/Pm6 +YaLkncTbpjer0zzEULfEngiHl8e8XrySeirmIZ7W1RPVA/k0f4d9rOVxhvtNM4es +WaEvCMxC1BOD5e7fX39hI4xjFNjecFSXPLR9RlbTxg0yQAjDfMJYghdNfUgfd8I0 +QP9UmSdLiUcCWJlZ5uF0UG+HNxcp/ML1z7GTLxYjuqC1gLA1giMD8Y7zJJbIsRKt +8ymtlkqoTkO+AMnQ0/Eno2yQ9ed7+guhYdpLuEvH2f+p5yEVtcrYmE/tiombs9Gq +twVTeSvmm8uLygQIdI0QeKnRjoM9qX9+5I1EkloB7vhTXSAQ+oECggEBAPuDyEgq +B2etxpvveDKOatuRimC+oWQ7eyp6NA8BOaHw+1OgTGPE+807i4/RPlhT9pbTuG16 +/unH8PnRXijtYEeQdFck9TCYjqwJlThZdokg30g827U6K+UECqd5ffejx9cnRpxu +Uke+AfMLdzG7G3EJlGoG76JZyKmow1JKzPhL7qa9YQRWA9dxC+vMjKakWf2Y8OSq +tkukYRpbn7VC99v5J8vJsNVFXu419N7h0bj2yQ6t64N/ybPWVfjp5xzc9rsNN14f +j1HoeqX/xw3MSUMjJol1L1V6+kHBhws0JsWFnGma+LTnDj4RE8HzohTlQv7Bhsgz +2qlW3gizrQEn+FECggEBANLjz4SX0eYF37pbZ1XReqezb9LP8oXiHn8UZpKlRpcF +DmaoSa9vcEySjwEq3oiR3Nzny46zLfAAJ7O3K2TI4AS/zcTmQw2G4+WjRf8tTq2x +A8SNq5E6p5bbimJC+80cVVfFAGukeQy9149ZW4ldfYTrk+821o5lBXmo9EqyrbqI +Nrt/EezSHr/Yai9zSV/VnLZ27nvW7vFlNHqbMGwhTHBY8eX6SEdIsobdjsdGdrUn +i331ImodBJ5/3H6OdNGHUbrizzn8Jm8CgZHkA87a9ON8eKHQ/FOb/Md82qDghnQR +LfBcoOac357Nprc4F/YGE4MCjXgLmGrgzMkQ0Fwcp80CggEBAM7fwQ/iSf70R2Uh +XhsvWyNInaofgj4gUplItKMW3eGehgpt0gdKEdboQE3FzOL4BN5gPNUIEr4Vr9a7 +aBh/zu5uGdNH2cjj4o4Mv8j+hOobuKwBKrHwrAQOA/lmi77x3sDQVFr8vv61gYL4 +jkzAWrzqJUHkfJxr/wnVfvqj/d3JDv3kzPS1DynYmPaVY6b5je9yKcnbxF+JUDlO +3ZlJAPfVAu+y8JkrGv8SMFxXH5pkmlFRqmKZ7DzYchRvx6HM+cA3CbCIgujbMG5z +aLWnrybitaLgWVOU+Fy3oq0Lc0yKLnIKfsDFP8i7YSXpkAph3G4Qnhzz0cnxYmWD +7CwERVECggEAaFVKalfOAVXwnLrxwbRUUTll3k8AthnraoWGRZC8/qQCvukNI10n +msp7M2GpHLnFIgkPXPbqiC0bdz7smf0DT3Yw7/PXQo70mryPObKJlUbZDVnlgoEZ +Pno42Wo4Nv6Ifla5YYfKV3JofcQAlFILckI2OwfPWD1EWy8qRPZnGryfD13LWXWO +vuzrg7QundoJoP/v9pacOhMOxoWWjDhhH8fxTQzoy1N891oPdCk5O2BoE5W+Q+89 +RMkPJhGGW87tsV7alN5ZiVwdDDdZZvJOa2k+KRhCbX7jrTHo2+SYwD1rk9nPxKfh +vigSDd0ThaT17D/MC5L5Ag9bYTIPUzLeFQKCAQEAq6RjI5A3Xppq9OFziOjgrUGv +2/xIH1NH7hdqGk5V+QRYQdD7Vd9wnF4f0CpIYTR55Mcud4amL3mHcR2IdjhJ4wcL +0VnSghllTdzO9dcDQ3cigIkzdikGoC+xPQRXMpt3sWS3BYyJZmjsUG1+TgPOZZeb +DInfb96I9euapu9meSrwzYy7R21eFfmVqqIaVkDv4fYfUiJZoSA9JygYul3jMt4p +rS7cdWaDaR/EX3aTA1S9S331CFwhzRYC5cj6t+Qz5SIH9czmHFH6STfIvYtkxGvC +GtROM9ZeDkO+/LwKbQlkbjuazbPCSWy5/163bPbK1w7PA2Ae7jMaJYtm3wYzQQ== +-----END RSA PRIVATE KEY----- diff --git a/tests/smoke/bare-metal/fake-creds/client.crt b/tests/smoke/bare-metal/fake-creds/client.crt new file mode 100644 index 0000000000..8f26ebab3b --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/client.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEYDCCAkigAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwEjEQMA4GA1UEAwwHZmFr +ZS1jYTAeFw0xNzAyMDEyMjEzMjVaFw0xODAyMDEyMjEzMjVaMBYxFDASBgNVBAMM +C2Zha2UtY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr8S7 +x/tAS6W+aRW3X833OvNfxXjUJAiRkUV85Raln7tqVcTG/9iyRhWgNpUn/WU1/3qV +obto4ZCURIwoQh0kWk8io1lafZJ+S6Znm3+0TKo7u6QMavolJyetsOQkT/bIoZ73 +09fhk4Vu9GILjtZtxV7GDb4WqR9R7z77nYTdHMio/BQVk+Xg6rkOsMRyoR+B9JHG +n9mvXLZSi8Q+3ABtsN6flPt7mTkhFFFvTgWxtzgVbeORT/uFxIV/IMjtGseUIzvF +GUQP6KCyCJb3Kp4rxSxIbi35mFqEWXjB7BVT/0pjx1mc5tSvGuFl7G4N/MmGe3Zq +ZCF4FalpiPGAInKrWQIDAQABo4G7MIG4MAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB +BAQDAgeAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg +Q2VydGlmaWNhdGUwHQYDVR0OBBYEFNZOj+0OOvhOFEtGGriZrPVCSzc3MB8GA1Ud +IwQYMBaAFNGPoXTjJnHjG2zMpjSg/9vNO/trMA4GA1UdDwEB/wQEAwIF4DATBgNV +HSUEDDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQsFAAOCAgEAiiGHlmPI6RlJQq7/ +z/1i0vFArDbnc2mwBf3pqrDPyqx1EBx7V3Tsm38TNMZyHaz0IyPUDvRPn10UYXui +2ZGseauwU/PmvFNofxVbG0Dc55lOoxl31520K0h9cWxVHcYzUxPndQ1pltYkXiMm +/596LHkJ+unMJszDVhAIOmc0PgECtGH1VG6EoTTFlMu7VJekKInkYNow4Q6cAVcr +11F4meOs0DMZgzfeUjSnsKG7KsLHfr5bLw6FEEzobgtI2sXVMOJi+ypd3zTY+ACq +oRt6wkRFCUoEgap7SG6B2TwHPGe15VIFZJtcnOZqHdrnfJLVROPnA4dYhJVJj1v1 +9JFH/T6EIi6nIqnrlX+10zaatpzq2+AFX8LiWpr7C7S99LgH3cnFdssfmlqoG82t +3BshYpDrIw1f72zy8+RCkK52OdjNpDoVwubwz6i8jldzoENqmsioyetyaVfe9GGH +UdEPrUZ4BHLeGPjHclOPVEhjVBZuofQ/GgM2gmCUdn5tcVLjnIeLAv/sQXwkMxIe +4m9QcPrxVAKOlDr9LhB0mVPr2kfc4yI/wYWEe+CniwcuvxJiOmjsyrENxfaFY30r +QspTSDVt8hVfVISzpuEchtLVjuRO/ESpmeOF1rRTc1qL/CjetmidkedDm64EZjyK +jyXQv9IZPMTwOndF6AVLH7l1F0E= +-----END CERTIFICATE----- diff --git a/tests/smoke/bare-metal/fake-creds/client.key b/tests/smoke/bare-metal/fake-creds/client.key new file mode 100644 index 0000000000..da8c408638 --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAr8S7x/tAS6W+aRW3X833OvNfxXjUJAiRkUV85Raln7tqVcTG +/9iyRhWgNpUn/WU1/3qVobto4ZCURIwoQh0kWk8io1lafZJ+S6Znm3+0TKo7u6QM +avolJyetsOQkT/bIoZ7309fhk4Vu9GILjtZtxV7GDb4WqR9R7z77nYTdHMio/BQV +k+Xg6rkOsMRyoR+B9JHGn9mvXLZSi8Q+3ABtsN6flPt7mTkhFFFvTgWxtzgVbeOR +T/uFxIV/IMjtGseUIzvFGUQP6KCyCJb3Kp4rxSxIbi35mFqEWXjB7BVT/0pjx1mc +5tSvGuFl7G4N/MmGe3ZqZCF4FalpiPGAInKrWQIDAQABAoIBAQCR/OQ+0JdxfWNu +YqQhBbA/nV7BZH9GwnstXrrCiBHeXsqOHFdwruo7PcEJNM+3LnYwEP/xCfityOjt +GkBh0VSdUbciV5fKTn9pk/ff9qypNIdSbYoG3Gc5Y0JndsYWSJIRczjCEj+AyMYE +Yt7Yr48S7ImxZl3p8GKcRQK1rWH9geg4cyCPisbaDSfjJbYh5yLk/2wsxGBRM3gg +CyJEbkJ/v107a1iThTGBgEgnFPP+FqZ2jlnfhBPVzuYggYyiMJuNtgDl7Vi7NLBe +2ueqq1UAT9LCpZNLJ8eYiDuyNHZtA7a2r3O/jTR4cvQy1xEjD3h4Es7olkAf/Lzu +6wuggbllAoGBANcqZyJtVxkGwHV9CWTWniTT7BNQ2ehYErkNKggMXl2AzOqEKzqn +IDRoBhiJKeAphdw/ccvqUEm9bUJD2QLpTJuMmUBkOwqMhATBXFrFCBX4PzGHYnC8 +6hEXjoUE6XhKdJEOgXTqrt31HDgj13GwAp/2DnsscFkC9co5+IW68sUjAoGBANEg +QvZYdI4Me6JxLXotyirpo57xjocvlo+uffws/YwBH8nK/op6am69zzMMOgUYA5Li +00WzfEXoyO+BdcbH28xYdBZT0CTkGlPM8IHuH+d/AwnEurxUElWZRRXSz6g17siM +KjBodqI8h+jQiQJuJ/zBJbOm3bUbpIt1Z+ROjstTAoGAWdAdVMWHQa8Lzv7uWOUt +Bfpf5IAvNUjuJ8hS7yEakrUc1BdvZAA29Skmwj8e967dbV4eRhv8f4tOfAaOIyT3 +EUbTAYnVC0Y0JTgBMPJluaXx2t7EPILewVuv5d5zBf8uQQ5pA0Ci1YtmyBhN6eqq +bdLroIagLseJiWxBTLEIfTkCgYEAjikXPC2fdhzoQuIbHy5Xe1p+PwNId3+TIzNk +M3RGG9F70YqsBGj5RzTC0JnkKyhK7aRCKOS9eyymw6HG9Y1RTpVmvPLW0O07NHJh +oIHGsHD4GMDijDm+iO/7Nb2sKlYXb79Qwr2Qv/LUFSEFsmA90KVgQsMRfhc/gQob +yOjaSz8CgYEAwr3aYp1CkKBXeUTNioLbyymhA4RqGPH/69F1NQ7froLXb152SzOV +jWcrt4ogRacgHb8thuTedrjUiJJLoWhQ3KqzSA2pI3tTLIxrJePiMMpt1Xb2z9l6 +Pikk0rvNVB/vrPeVjAdGY9TJC/vpz3om92DRDmUifu8rCFxIHE0GrQ0= +-----END RSA PRIVATE KEY----- diff --git a/tests/smoke/bare-metal/fake-creds/server.crt b/tests/smoke/bare-metal/fake-creds/server.crt new file mode 100644 index 0000000000..f4fd4f87fd --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/server.crt @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEtDCCApygAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwEjEQMA4GA1UEAwwHZmFr +ZS1jYTAeFw0xNzAyMDEyMjEzMjRaFw0xODAyMDEyMjEzMjRaMBYxFDASBgNVBAMM +C2Zha2Utc2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy4kG +FosBCFT5YZn2To2oYDT8c3LYwj39Xgm1N14ESfz5JfcVEB9/p20xrWyWlMB3W8y/ +QVRhoTdZQxPqzZzCcTtwLxOyAxr18iYD8HKYrDidpJj9xvq0NJWio+RbL0osMpv0 +kPVORwo0PxPwnYCCkV41aYRC+Iun4Lg3o1Dq8aFLo9prsF6XvXG8XDRzeSfmas9R +4UZIYH5GTocv//HNHIdR199crof6zwsSL0wA7DRZl2PvOEUjdkWvz+DTGVThK/y6 +EQ894wW+f+2u8QePH3FYJfXTDfLkOXBkGmIvrYd+gZDrrRPS934P1c8iEiip229Z +ydVERJNggSe8q3tKMQIDAQABo4IBDjCCAQowCQYDVR0TBAIwADARBglghkgBhvhC +AQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZl +ciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUbA7PtucmrSIHnVx7uirtZ0PCGQcwQgYD +VR0jBDswOYAU0Y+hdOMmceMbbMymNKD/2807+2uhFqQUMBIxEDAOBgNVBAMMB2Zh +a2UtY2GCCQCLl6tdJNjhZTAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwLQYDVR0RBCYwJIIUbWF0Y2hib3guZXhhbXBsZS5jb22CDG1hdGNoYm94 +LmZvbzANBgkqhkiG9w0BAQsFAAOCAgEAg/JBZsWmSRR1qe2Xvhby0sm0mPFHF4La +C7HcSAgSeo1buBstyZQ7DEu87mrlhLZyoPFs0AA6TX5wT2js2cv59D052edKVdxq +WBNyL4gjZKcBS4M48wSznVeReGiY6zXgPbRxQ8YxNp6YvVvqIPMTDXXI1VMNlQ7+ +u+3TfB9d+yiDYwXUxH5/J7GESHe0kT/4MaP/uGk8f/dvytExZXwpBJgA4TlmN/Cc +oNznht2uJbB8AsMNsAYjeCehAUyAT/N2OadLCdj87cw07vnUZ7TL2DvbIJc1aXus +KL2RGGDIXZVkeRwppI+L6etLY2H5XAEGe25x9gEiKPpay8pNhwHvs+qy0Whwuaoq +XxPQ3ByJ6eAL6BSyPSNjFpJswWQGEtLYicknwWs0gsnDIrqO1O0kZ7HamHAikMmw +qKBhOBcPclX7EQvO8KqjT8Jg4qKkWdtkPCtRDT3JGA+KX4e0B+4CRI3xKwJ9CYlc +lOY5rA21TBbNazAjQ8SQdLf5gFVD6sdYvUAR+XxAz9hcSal4p41IxUAE8grwnmdE +HJw7wNkDashfjOOqrC/1vE8jYv44yw/ogyfQia1jtsdO0q9jYw3wtqI2UoDLa2sM +whB8YEiEAbCVAmVopPqjE5WbhxPlvQ+iH0oD2WlCgHB5nGnjLcVPWJheYRd1du4t +7R3wf6OoHJ8= +-----END CERTIFICATE----- diff --git a/tests/smoke/bare-metal/fake-creds/server.key b/tests/smoke/bare-metal/fake-creds/server.key new file mode 100644 index 0000000000..25fd2423ec --- /dev/null +++ b/tests/smoke/bare-metal/fake-creds/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAy4kGFosBCFT5YZn2To2oYDT8c3LYwj39Xgm1N14ESfz5JfcV +EB9/p20xrWyWlMB3W8y/QVRhoTdZQxPqzZzCcTtwLxOyAxr18iYD8HKYrDidpJj9 +xvq0NJWio+RbL0osMpv0kPVORwo0PxPwnYCCkV41aYRC+Iun4Lg3o1Dq8aFLo9pr +sF6XvXG8XDRzeSfmas9R4UZIYH5GTocv//HNHIdR199crof6zwsSL0wA7DRZl2Pv +OEUjdkWvz+DTGVThK/y6EQ894wW+f+2u8QePH3FYJfXTDfLkOXBkGmIvrYd+gZDr +rRPS934P1c8iEiip229ZydVERJNggSe8q3tKMQIDAQABAoIBAQCkI9jZDTjD82M6 +JuPP7cW/8Hs2z4OnAj72O3X9NMDYuj5jM6O04PIdn53UciHtzFl1muUSVm51iSFf +SwyszKKylnCeRo/kmTWHFslOe9xL50yalRb0BPKXBaWn7lzrAjIqjkLpNCrwd3Yy ++G2vVxf1C7ifY55uupGkVGvqNTODsFKbEDAMJeWIucFnVfjdtxwtIb/62TDSNztB +WjaU0ySAOQaTeAwRgkIYm/yE3OtKTwRjvfkb5W2EWKqn4Q66H2nXal6shA2WMSB4 +AlED/MDfKoUBIXBmWrSo9V4NF/P920+4ozHd4dHN4QKuu1BR5rWuVK9p2Vf0RVfQ +M9rQuSxRAoGBAPm4m9XpRsQH85yvBujeMC+Z+qHdDt/f+V/6hMJNuYG4q0Vu4rYH +5VY8FJyBkjE4oNBWK+FuhhfccVWRLFeId/xSD5d0Xs67q2Asb2WgTNC9joWiY8Jp +GLPjS3HYfjngDLcvDMhtNPPwPRfmLN8P7OVsQj/W8i1arKBUDkEJDIr3AoGBANCn +INlWEfLBEBpcy20ZVKB0u5d8KRd/paLXu2aNL6qP/PPq+sEl763hJkpuCKCkyDkC +loWEAP2QnamBPVp6QuwQfCGDWmFZEzsAEjBoUckxpUR/eHuYG6RQ5z4AS/92QeLp +kVvMq1D0NstCRv3LvJfANbICf+Gp61eVTJRGyiIXAoGAJIPRDatptJpxDdUuDnkG +1L8QPgDj/upyPY2VQhA5cvV83c9ECVgIOd7ZKLVtmhLsuOoQyortwqlyTQ/ISxhi +axvYuc3DWzln6s00kMiq0s8U+wduzOuvOAahbok4k9/mHBceKGSHnK/hTxBSOV/L +xRRab5lPmkArA9qmpu3azqsCgYAUUZIjV/xbdW2KA6tTrnnNPlDeSZr+zNFmcDex +nuIba/YoEqPklzsl5Z0C9oAklKdi1duumNI33xNbZFiDLRDDSD+uS8pLXI9A2Zrr +19zPCOSdLQSglIlskOxT31eHQw7bd/9NvzqXoZCMbqHXKTXuWSthcFjfstpWJxXU +AqjAdQKBgQC5SscpSZY9a3Dmi6HQkNV7hrNTLLnpvGuVQPaJwfhV5Gc07qtbg7xW +ol8uAOpRxDoXI5KNJFu/eYbj++ms8IdG9xCW+zOPBnxZ96vuUMgDHNvslVL5V1Jy +bZ/01Vam+vIUcO2zDKimBRkDiz3BkyiHNOXMrn7bFFkWVauZihA+lw== +-----END RSA PRIVATE KEY----- diff --git a/tests/smoke/bare-metal/smoke.sh b/tests/smoke/bare-metal/smoke.sh new file mode 100755 index 0000000000..bf5abeec85 --- /dev/null +++ b/tests/smoke/bare-metal/smoke.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# This scripts brings up a bare-metal Tectonic cluster using VMs and +# containerized matchbox/dnsmasq servers. +# +# The following environment variables are expected: +# - BRANCH_NAME +# - BUILD_ID +# The script setups the environment and calls `make apply` from the repository +# root. +# +# Due to the assumptions made by this script, it is *not* safe to run multiple +# instances of it on a single host and the Terraform configuration must be +# matching the infrastructure. Notably: +# - matchbox is expected on 172.18.0.2, +# - three nodes are expected on 172.18.0.21 (master), 172.18.0.22 (worker), 172.18.0.23 (worker). +# +# This script requires the following packages on the host: +# - qemu-kvm +# - libvirt-bin +# - virt-manager +# - curl +# - unzip +# - policycoreutils. +set -xe + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT="$DIR/../../.." +BIN_DIR="$ROOT/bin_test" + +MATCHBOX_VERSION=v0.6.1 +KUBECTL_VERSION=v1.6.4 +TERRAFORM_VERSION=0.9.6 + +KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" +TERRAFORM_URL="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" + +export VM_DISK='20' +export VM_MEMORY='2048' +export ASSETS_DIR="${ASSETS_DIR:-/tmp/matchbox/assets}" + +main() { + if [ -z "${BRANCH_NAME}" ] || [ -z "${BUILD_ID}" ]; then + echo "\$BRANCH_NAME, \$BUILD_ID are required" + return 1 + fi + if [ -z "$1" ]; then + echo "$0 " + return 1 + fi + + echo "Installing required binaries" + install + + echo "Cleanup testing environment" + cleanup &>/dev/null + trap kill_terraform_and_cleanup EXIT + + echo "Setting up configuration and environment" + configure $1 + setup + + echo "Starting matchbox" + (cd ${ROOT}/matchbox && sudo -S -E ./scripts/devnet create) + echo "Waiting for matchbox..." + until $(curl --silent --fail -k http://matchbox.example.com:8080 > /dev/null); do + echo "Waiting for matchbox..." + sleep 5 + done + + echo "Starting Terraform" + (cd ${ROOT} && make apply || kill $$) & # Self-destruct and trigger trap on failure + TERRAFORM_PID=$! + sleep 15 + + echo "Starting QEMU/KVM nodes" + (cd ${ROOT}/matchbox && sudo -E ./scripts/libvirt create) + + echo "Waiting for Kubernetes/Tectonic cluster to be up and running:" + cluster_up + + echo "Running Go sanity tests" + test_cluster + + echo "SUCCESS: Tectonic bare-metal cluster came up!" + cleanup +} + +install() { + mkdir -p $BIN_DIR + export PATH=$BIN_DIR:$PATH + + echo "Installing kubectl" + curl -L -o ${BIN_DIR}/kubectl ${KUBECTL_URL} + chmod +x ${BIN_DIR}/kubectl + + echo "Installing Terraform" + curl ${TERRAFORM_URL} | funzip > $BIN_DIR/terraform + sudo chmod +x $BIN_DIR/terraform + + echo "Installing matchbox" + (cd ${ROOT}/ && rm -rf matchbox && git clone https://github.com/coreos/matchbox) + (cd ${ROOT}/matchbox && git checkout $MATCHBOX_VERSION) +} + +setup() { + echo "Copying matchbook test credentials" + cp ${DIR}/fake-creds/{ca.crt,server.crt,server.key} ${ROOT}/matchbox/examples/etc/matchbox + + if [ ! -d $ASSETS_DIR/coreos/$COREOS_VERSION ]; then + echo "Downloading CoreOS image" + ${ROOT}/matchbox/scripts/get-coreos $COREOS_CHANNEL $COREOS_VERSION $ASSETS_DIR + fi + + echo "Configuring ssh-agent" + eval `ssh-agent -s` + chmod 600 ${ROOT}/matchbox/tests/smoke/fake_rsa + ssh-add ${ROOT}/matchbox/tests/smoke/fake_rsa + + echo "Setting up the metal0 bridge" + sudo mkdir -p /etc/rkt/net.d + sudo bash -c 'cat > /etc/rkt/net.d/20-metal.conf << EOF + { + "name": "metal0", + "type": "bridge", + "bridge": "metal0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.18.0.0/24", + "routes" : [ { "dst" : "0.0.0.0/0" } ] + } + } +EOF' + + echo "Setting up DNS" + if ! grep -q "172.18.0.3" /etc/resolv.conf; then + echo "nameserver 172.18.0.3" | cat - /etc/resolv.conf | sudo tee /etc/resolv.conf >/dev/null + fi +} + +configure() { + export PLATFORM=metal + export CLUSTER="tf-${PLATFORM}-${BRANCH_NAME}-${BUILD_ID}" + export TF_VAR_tectonic_cluster_name=$(echo ${CLUSTER} | awk '{print tolower($0)}') + + CONFIG=${DIR}/$1 + make localconfig + ln -sf ${CONFIG} ${ROOT}/build/${CLUSTER}/terraform.tfvars + + COREOS_CHANNEL=$(awk -F "=" '/^tectonic_cl_channel/ {gsub(/[ \t"]/, "", $2); print $2}' ${CONFIG}) + COREOS_VERSION=$(awk -F "=" '/^tectonic_metal_cl_version/ {gsub(/[ \t"]/, "", $2); print $2}' ${CONFIG}) + + export TEST_KUBECONFIG=${ROOT}/build/${CLUSTER}/generated/auth/kubeconfig +} + +cleanup() { + set +e + + # Kill any remaining VMs. + (cd ${ROOT}/matchbox && sudo ./scripts/libvirt destroy) + + # Reset rkt pods and CNI entirely, to avoid IP conflict due to leakage bug. + for p in `sudo rkt list | tail -n +2 | awk '{print $1}'`; do sudo rkt stop --force $p; done + sudo rkt gc --grace-period=0s + + for ns in `ip netns l | grep -o -E '^[[:alnum:]]+'`; do sudo ip netns del $ns; done + sudo ip l del metal0 + for veth in `ip l show | grep -oE 'veth[^@]+'`; do sudo ip l del $veth; done + + sudo rm -Rf /var/lib/cni/networks/* + sudo rm -Rf /var/lib/rkt/* + sudo rm -f /etc/rkt/net.d/20-metal.conf + + # Reset DNS. + cat /etc/resolv.conf | grep -v 172.18.0.3 | sudo tee /etc/resolv.conf + + # Reset failed units (i.e. matchbox, dnsmasq which we just killed). + sudo systemctl reset-failed + + set -e +} + +kill_terraform_and_cleanup() { + echo "Killing Terraform" + kill ${TERRAFORM_PID} || true + + echo "WARNING: Cleanup is temporarily disabled on failure for debugging purposes. Next job will clean at startup." + #echo "Cleanup testing environment" + #cleanup +} + +kubelet_up() { + curl --silent --fail -m 1 http://$1:10255/healthz > /dev/null +} + +cluster_up() { + echo "Waiting for Kubelets to start..." + until kubelet_up "node1.example.com" \ + && kubelet_up "node2.example.com" \ + && kubelet_up "node3.example.com" + do + sleep 15 + echo "Waiting for Kubelets to start..." + done + + echo "$(readyNodes) of 3 nodes are Ready..." + until [[ "$(readyNodes)" == "3" ]]; do + sleep 5 + echo "$(readyNodes) of 3 nodes are Ready..." + done + + echo "List of nodes:" + k8s get nodes + + sleep 5 + until [[ "$(readyPods)" == "$(podCount)" && "$(readyPods)" -gt "0" ]]; do + sleep 15 + echo "$(readyPods) / $(podCount) pods are Running..." + k8s get pods --all-namespaces || true + done + + echo "List of pods:" + k8s get pods --all-namespaces || true + + echo "Waiting for Tectonic Console..." + until $(curl --silent --fail -k https://tectonic.example.com > /dev/null); do + echo "Waiting for Tectonic Console..." + k8s get pods --all-namespaces || true + sleep 15 + done +} + +k8s() { + ${BIN_DIR}/kubectl --kubeconfig=${TEST_KUBECONFIG} "$@" +} + +# ready nodes returns the number of Ready Kubernetes nodes +readyNodes() { + k8s get nodes -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l +} + +# ready pods returns the number of Running pods +readyPods() { + k8s get pods --all-namespaces -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E True | wc -l +} + +# podCount returns the number of pods +podCount() { + k8s get pods --all-namespaces -o template --template='{{range .items}}{{range .status.conditions}}{{if eq .type "Ready"}}{{.}}{{end}}{{end}}{{end}}' | grep -o -E status | wc -l +} + +test_cluster() { + MASTER_COUNT=$(grep tectonic_master_count "$CONFIG" | awk -F "=" '{gsub(/"/, "", $2); print $2}') + WORKER_COUNT=$(grep tectonic_worker_count "$CONFIG" | awk -F "=" '{gsub(/"/, "", $2); print $2}') + export NODE_COUNT=$(( MASTER_COUNT + WORKER_COUNT )) + installer/bin/sanity -test.v -test.parallel=1 +} + +main "$@" diff --git a/tests/smoke/bare-metal/vars/metal.tfvars b/tests/smoke/bare-metal/vars/metal.tfvars new file mode 100644 index 0000000000..10321197ef --- /dev/null +++ b/tests/smoke/bare-metal/vars/metal.tfvars @@ -0,0 +1,138 @@ +tectonic_metal_matchbox_http_url = "http://matchbox.example.com:8080" +tectonic_metal_matchbox_rpc_endpoint = "matchbox.example.com:8081" + +tectonic_metal_matchbox_client_cert = <