From cc8a3e7657c2f84b9acf3b1353ec33504f6e016d Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Fri, 6 Nov 2015 11:37:10 +0100 Subject: [PATCH] Updating build process to match orign. 1. Added make release 2. Builds are placed in _output/local/bin/$(platform) directory 3. Added i386 binaries in release --- CONTRIBUTING.md | 4 +- Makefile | 21 ++++-- hack/build-cross.sh | 16 ++--- hack/build-release.sh | 25 ++++--- hack/common.sh | 148 ++++++++++++++++++++++++-------------- hack/extract-release.sh | 23 ++++++ hack/test-stirunimage.sh | 10 +-- hack/util.sh | 72 ++++++++++++++++++- images/release/Dockerfile | 7 +- 9 files changed, 230 insertions(+), 96 deletions(-) create mode 100755 hack/extract-release.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 877c50296..2da6f04aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,12 +93,12 @@ If you are not updating packages you should not need godep installed. ## Building a Release -To build a S2I release you run the `hack/build-release.sh` script on a system with Docker, +To build a S2I release you run `make release` on a system with Docker, which will create a build environment image and then execute a cross platform Go build within it. The build output will be copied to `_output/releases` as a set of tars containing each version. 1. Create a new git tag `git tag v0.X.X -a -m "v0.X.X" HEAD` 2. Push the tag to GitHub `git push origin --tags` where `origin` is `github.com/openshift/source-to-image.git` -4. Run `hack/build-release.sh` +4. Run `make release` 5. Upload the binary artifacts generated by that build to GitHub release page. 6. Send an email to the dev list, including the important changes prior to the release. diff --git a/Makefile b/Makefile index e5dbd258f..7465fa019 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ # check: Run tests. # test: Run tests. # clean: Clean up. +# release: Build release. OUT_DIR = _output OUT_PKG_DIR = Godeps/_workspace/pkg @@ -24,7 +25,6 @@ all build: hack/build-go.sh .PHONY: all build - # Verify code is properly organized. # # Example: @@ -35,7 +35,6 @@ verify: build hack/verify-govet.sh || true .PHONY: verify - # Install travis dependencies # # Example: @@ -44,21 +43,22 @@ install-travis: hack/install-tools.sh .PHONY: install-travis -# Build and run tests. +# Build and run unit tests # # Args: # WHAT: Directory names to test. All *_test.go files under these # directories will be run. If not specified, "everything" will be tested. # TESTS: Same as WHAT. # GOFLAGS: Extra flags to pass to 'go' when building. +# TESTFLAGS: Extra flags that should only be passed to hack/test-go.sh # # Example: # make check # make test # make check WHAT=pkg/build GOFLAGS=-v -check test: verify - STI_COVER="-cover -covermode=atomic" STI_RACE="-race" hack/test-go.sh $(WHAT) $(TESTS) -.PHONY: check test +check: + hack/test-go.sh $(WHAT) $(TESTS) $(TESTFLAGS) +.PHONY: check # Remove all build artifacts. # @@ -67,3 +67,12 @@ check test: verify clean: rm -rf $(OUT_DIR) $(OUT_PKG_DIR) .PHONY: clean + +# Build the release. +# +# Example: +# make release +release: clean + hack/build-release.sh + hack/extract-release.sh +.PHONY: release diff --git a/hack/build-cross.sh b/hack/build-cross.sh index b59b6082a..607f2cdae 100755 --- a/hack/build-cross.sh +++ b/hack/build-cross.sh @@ -6,21 +6,19 @@ set -o errexit set -o nounset set -o pipefail +STARTTIME=$(date +%s) STI_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${STI_ROOT}/hack/common.sh" +source "${STI_ROOT}/hack/util.sh" +sti::log::install_errexit # Build the primary for all platforms STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}") sti::build::build_binaries "${STI_CROSS_COMPILE_TARGETS[@]}" -# Build image binaries for a subset of platforms. Image binaries are currently -# linux-only, and are compiled with flags to make them static for use in Docker -# images "FROM scratch". -STI_BUILD_PLATFORMS=("${STI_IMAGE_COMPILE_PLATFORMS[@]-}") -CGO_ENABLED=0 STI_GOFLAGS="-a" sti::build::build_binaries "${STI_IMAGE_COMPILE_TARGETS[@]-}" - # Make the primary release. STI_RELEASE_ARCHIVE="source-to-image" -STI_RELEASE_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}") -STI_RELEASE_BINARIES=("${STI_CROSS_COMPILE_BINARIES[@]}") -sti::build::place_bins +STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}") +sti::build::place_bins "${STI_CROSS_COMPILE_BINARIES[@]}" + +ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret" diff --git a/hack/build-release.sh b/hack/build-release.sh index d5e9ceffc..5a48bef03 100755 --- a/hack/build-release.sh +++ b/hack/build-release.sh @@ -7,8 +7,11 @@ set -o errexit set -o nounset set -o pipefail +STARTTIME=$(date +%s) STI_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${STI_ROOT}/hack/common.sh" +source "${STI_ROOT}/hack/util.sh" +sti::log::install_errexit # Go to the top of the tree. cd "${STI_ROOT}" @@ -20,30 +23,26 @@ docker build -q --tag openshift/sti-release "${STI_ROOT}/images/release" context="${STI_ROOT}/_output/buildenv-context" # Clean existing output. -rm -rf "${STI_ROOT}/_output/local/releases" -rm -rf "${STI_ROOT}/_output/local/go/bin" +rm -rf "${STI_LOCAL_RELEASEPATH}" rm -rf "${context}" mkdir -p "${context}" -mkdir -p "${STI_ROOT}/_output/local" +mkdir -p "${STI_OUTPUT}" # Generate version definitions. +# You can commit a specific version by specifying STI_GIT_COMMIT="" prior to build sti::build::get_version_vars sti::build::save_version_vars "${context}/sti-version-defs" +echo "++ Building release ${STI_GIT_VERSION}" + # Create the input archive. -git archive --format=tar -o "${context}/archive.tar" HEAD +git archive --format=tar -o "${context}/archive.tar" "${STI_GIT_COMMIT}" tar -rf "${context}/archive.tar" -C "${context}" sti-version-defs gzip -f "${context}/archive.tar" # Perform the build and release in Docker. cat "${context}/archive.tar.gz" | docker run -i --cidfile="${context}/cid" openshift/sti-release -docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_ROOT}/_output/local" -echo "${STI_GIT_COMMIT}" > "${STI_ROOT}/_output/local/releases/.commit" +docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_OUTPUT}" +echo "${STI_GIT_COMMIT}" > "${STI_LOCAL_RELEASEPATH}/.commit" -# Copy the linux release archives release back to the local _output/local/go/bin directory. -sti::build::detect_local_release_tars "linux" - -mkdir -p "${STI_LOCAL_BINPATH}" -tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_LOCAL_BINPATH}" - -sti::build::make_binary_symlinks +ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret" diff --git a/hack/common.sh b/hack/common.sh index aba0484e4..ace746f00 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -18,8 +18,8 @@ STI_ROOT=$( STI_OUTPUT_SUBPATH="${STI_OUTPUT_SUBPATH:-_output/local}" STI_OUTPUT="${STI_ROOT}/${STI_OUTPUT_SUBPATH}" STI_OUTPUT_BINPATH="${STI_OUTPUT}/bin" -STI_LOCAL_BINPATH="${STI_ROOT}/_output/local/go/bin" -STI_LOCAL_RELEASEPATH="${STI_ROOT}/_output/local/releases" +STI_LOCAL_BINPATH="${STI_OUTPUT}/go/bin" +STI_LOCAL_RELEASEPATH="${STI_OUTPUT}/releases" readonly STI_GO_PACKAGE=github.com/openshift/source-to-image readonly STI_GOPATH="${STI_OUTPUT}/go" @@ -28,6 +28,7 @@ readonly STI_CROSS_COMPILE_PLATFORMS=( linux/amd64 darwin/amd64 windows/amd64 + linux/386 ) readonly STI_CROSS_COMPILE_TARGETS=( cmd/s2i @@ -44,6 +45,10 @@ readonly STI_BINARY_SYMLINKS=( readonly STI_BINARY_COPY=( sti ) +readonly STI_BINARY_RELEASE_WINDOWS=( + sti.exe + s2i.exe +) # sti::build::binaries_from_targets take a list of build targets and return the # full go package to be built @@ -78,32 +83,7 @@ sti::build::build_binaries() { local version_ldflags version_ldflags=$(sti::build::ldflags) - # Use eval to preserve embedded quoted strings. - local goflags - eval "goflags=(${STI_GOFLAGS:-})" - - local -a targets=() - local arg - for arg; do - if [[ "${arg}" == -* ]]; then - # Assume arguments starting with a dash are flags to pass to go. - goflags+=("${arg}") - else - targets+=("${arg}") - fi - done - - if [[ ${#targets[@]} -eq 0 ]]; then - targets=("${STI_ALL_TARGETS[@]}") - fi - - local -a platforms=("${STI_BUILD_PLATFORMS[@]:+${STI_BUILD_PLATFORMS[@]}}") - if [[ ${#platforms[@]} -eq 0 ]]; then - platforms=("$(sti::build::host_platform)") - fi - - local binaries - binaries=($(sti::build::binaries_from_targets "${targets[@]}")) + sti::build::export_targets "$@" local platform for platform in "${platforms[@]}"; do @@ -117,6 +97,38 @@ sti::build::build_binaries() { ) } +# Generates the set of target packages, binaries, and platforms to build for. +# Accepts binaries via $@, and platforms via STI_BUILD_PLATFORMS, or defaults to +# the current platform. +sti::build::export_targets() { + # Use eval to preserve embedded quoted strings. + local goflags + eval "goflags=(${STI_GOFLAGS:-})" + + targets=() + local arg + for arg; do + if [[ "${arg}" == -* ]]; then + # Assume arguments starting with a dash are flags to pass to go. + goflags+=("${arg}") + else + targets+=("${arg}") + fi + done + + if [[ ${#targets[@]} -eq 0 ]]; then + targets=("${STI_ALL_TARGETS[@]}") + fi + + binaries=($(sti::build::binaries_from_targets "${targets[@]}")) + + platforms=("${STI_BUILD_PLATFORMS[@]:+${STI_BUILD_PLATFORMS[@]}}") + if [[ ${#platforms[@]} -eq 0 ]]; then + platforms=("$(sti::build::host_platform)") + fi +} + + # Takes the platform name ($1) and sets the appropriate golang env variables # for that platform. sti::build::set_platform_envs() { @@ -167,7 +179,7 @@ sti::build::setup_env() { sti::build::create_gopath_tree if [[ -z "$(which go)" ]]; then - echo < darwin_amd64. local platform_src="/${platform//\//_}" @@ -235,21 +267,19 @@ sti::build::place_bins() { # Create an array of binaries to release. Append .exe variants if the platform is windows. local -a binaries=() - local binary - for binary in "${STI_RELEASE_BINARIES[@]}"; do - binaries+=("${binary}") + for binary in "${targets[@]}"; do + binary=$(basename $binary) if [[ $platform == "windows/amd64" ]]; then binaries+=("${binary}.exe") + else + binaries+=("${binary}") fi done - # Copy only the specified release binaries to the shared STI_OUTPUT_BINPATH. - local -a includes=() + # Move the specified release binaries to the shared STI_OUTPUT_BINPATH. for binary in "${binaries[@]}"; do - includes+=("--include=${binary}") + mv "${full_binpath_src}/${binary}" "${STI_OUTPUT_BINPATH}/${platform}/" done - find "${full_binpath_src}" -maxdepth 1 -type f -exec \ - rsync "${includes[@]}" --exclude="*" -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \; # If no release archive was requested, we're done. if [[ "${STI_RELEASE_ARCHIVE-}" == "" ]]; then @@ -258,8 +288,9 @@ sti::build::place_bins() { # Create a temporary bin directory containing only the binaries marked for release. local release_binpath=$(mktemp -d sti.release.${STI_RELEASE_ARCHIVE}.XXX) - find "${full_binpath_src}" -maxdepth 1 -type f -exec \ - rsync "${includes[@]}" --exclude="*" -pt {} "${release_binpath}" \; + for binary in "${binaries[@]}"; do + cp "${STI_OUTPUT_BINPATH}/${platform}/${binary}" "${release_binpath}/" + done # Create binary copies where specified. local suffix="" @@ -275,10 +306,17 @@ sti::build::place_bins() { # Create the release archive. local platform_segment="${platform//\//-}" - local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz" - - echo "++ Creating ${archive_name}" - tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" . + if [[ $platform == "windows/amd64" ]]; then + local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.zip" + echo "++ Creating ${archive_name}" + for file in "${STI_BINARY_RELEASE_WINDOWS[@]}"; do + zip "${STI_LOCAL_RELEASEPATH}/${archive_name}" -qj "${release_binpath}/${file}" + done + else + local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz" + echo "++ Creating ${archive_name}" + tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" . + fi rm -rf "${release_binpath}" done ) @@ -287,15 +325,13 @@ sti::build::place_bins() { # sti::build::make_binary_symlinks makes symlinks for the sti # binary in _output/local/go/bin sti::build::make_binary_symlinks() { - local host_platform - host_platform=$(sti::build::host_platform) - - if [[ -f "${STI_LOCAL_BINPATH}/s2i" ]]; then + platform=$(sti::build::host_platform) + if [[ -f "${STI_OUTPUT_BINPATH}/${platform}/s2i" ]]; then for linkname in "${STI_BINARY_SYMLINKS[@]}"; do - if [[ $host_platform == "windows/amd64" ]]; then - cp "${STI_LOCAL_BINPATH}/s2i.exe" "${STI_LOCAL_BINPATH}/${linkname}.exe" + if [[ $platform == "windows/amd64" ]]; then + cp s2i "${STI_OUTPUT_BINPATH}/${platform}/${linkname}.exe" else - ln -sf "${STI_LOCAL_BINPATH}/s2i" "${STI_LOCAL_BINPATH}/${linkname}" + ln -sf s2i "${STI_OUTPUT_BINPATH}/${platform}/${linkname}" fi done fi @@ -311,7 +347,15 @@ sti::build::make_binary_symlinks() { sti::build::detect_local_release_tars() { local platform="$1" - local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}-*) + if [[ ! -d "${STI_LOCAL_RELEASEPATH}" ]]; then + echo "There are no release artifacts in ${STI_LOCAL_RELEASEPATH}" + exit 2 + fi + if [[ ! -f "${STI_LOCAL_RELEASEPATH}/.commit" ]]; then + echo "There is no release .commit identifier ${STI_LOCAL_RELEASEPATH}" + exit 2 + fi + local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}*) if [[ $(echo "${primary}" | wc -l) -ne 1 ]]; then echo "There should be exactly one ${platform} primary tar in $STI_LOCAL_RELEASEPATH" exit 2 diff --git a/hack/extract-release.sh b/hack/extract-release.sh new file mode 100755 index 000000000..18f494b6a --- /dev/null +++ b/hack/extract-release.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script extracts a valid release tar into _output/releases. It requires hack/build-release.sh +# to have been executed + +set -o errexit +set -o nounset +set -o pipefail + +STI_ROOT=$(dirname "${BASH_SOURCE}")/.. +source "${STI_ROOT}/hack/common.sh" + +# Go to the top of the tree. +cd "${STI_ROOT}" + +# Copy the linux release archives release back to the local _output/local/bin/linux/amd64 directory. +# TODO: support different OS's? +sti::build::detect_local_release_tars "linux-amd64" + +mkdir -p "${STI_OUTPUT_BINPATH}/linux/amd64" +tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_OUTPUT_BINPATH}/linux/amd64" + +sti::build::make_binary_symlinks diff --git a/hack/test-stirunimage.sh b/hack/test-stirunimage.sh index 5209a96d4..af8de0fc1 100755 --- a/hack/test-stirunimage.sh +++ b/hack/test-stirunimage.sh @@ -35,8 +35,8 @@ fi trap cleanup EXIT SIGINT -echo "Running 'sti build --run=true ...'" -s2i build git://github.com/bparees/openshift-jee-sample openshift/wildfly-8-centos test-jee-app --run=true &> "${STI_ROOT}/hack/sti-run.log" & +echo "Running 's2i build --run=true ...'" +s2i build git://github.com/bparees/openshift-jee-sample openshift/wildfly-81-centos7 test-jee-app --run=true &> "${STI_ROOT}/hack/sti-run.log" & export STI_PID=$! TIME_SEC=1000 TIME_MIN=$((60 * $TIME_SEC)) @@ -48,8 +48,8 @@ set +e while [[ $(time_now) -lt $expire ]]; do grep "as a result of the --run=true option" "${STI_ROOT}/hack/sti-run.log" if [ $? -eq 0 ]; then - echo "[INFO] Success running command sti --run=true" - + echo "[INFO] Success running command s2i --run=true" + # use sigint so that sti post processing will remove docker container kill -2 "${STI_PID}" export KILLDONE="killed" @@ -68,7 +68,7 @@ while [[ $(time_now) -lt $expire ]]; do sleep 1 done -echo "[INFO] Problem with sti --run=true, dumping ${STI_ROOT}/hack/sti-run.log" +echo "[INFO] Problem with s2i --run=true, dumping ${STI_ROOT}/hack/sti-run.log" cat "${STI_ROOT}/hack/sti-run.log" set -e exit 1 diff --git a/hack/util.sh b/hack/util.sh index c4c39ff4a..8ee8c5eac 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -2,15 +2,81 @@ # Provides simple utility functions +# Handler for when we exit automatically on an error. +# Borrowed from https://gist.github.com/ahendrix/7030300 +sti::log::errexit() { + local err="${PIPESTATUS[@]}" + + # If the shell we are in doesn't have errexit set (common in subshells) then + # don't dump stacks. + set +o | grep -qe "-o errexit" || return + + set +o xtrace + local code="${1:-1}" + sti::log::error_exit "'${BASH_COMMAND}' exited with status $err" "${1:-1}" 1 +} + +sti::log::install_errexit() { + # trap ERR to provide an error handler whenever a command exits nonzero this + # is a more verbose version of set -o errexit + trap 'sti::log::errexit' ERR + + # setting errtrace allows our ERR trap handler to be propagated to functions, + # expansions and subshells + set -o errtrace +} + +# Print out the stack trace +# +# Args: +# $1 The number of stack frames to skip when printing. +sti::log::stack() { + local stack_skip=${1:-0} + stack_skip=$((stack_skip + 1)) + if [[ ${#FUNCNAME[@]} -gt $stack_skip ]]; then + echo "Call stack:" >&2 + local i + for ((i=1 ; i <= ${#FUNCNAME[@]} - $stack_skip ; i++)) + do + local frame_no=$((i - 1 + stack_skip)) + local source_file=${BASH_SOURCE[$frame_no]} + local source_lineno=${BASH_LINENO[$((frame_no - 1))]} + local funcname=${FUNCNAME[$frame_no]} + echo " $i: ${source_file}:${source_lineno} ${funcname}(...)" >&2 + done + fi +} + +# Log an error and exit. +# Args: +# $1 Message to log with the error +# $2 The error code to return +# $3 The number of stack frames to skip when printing. +sti::log::error_exit() { + local message="${1:-}" + local code="${2:-1}" + local stack_skip="${3:-0}" + stack_skip=$((stack_skip + 1)) + + local source_file=${BASH_SOURCE[$stack_skip]} + local source_line=${BASH_LINENO[$((stack_skip - 1))]} + echo "!!! Error in ${source_file}:${source_line}" >&2 + [[ -z ${1-} ]] || { + echo " ${1}" >&2 + } + + sti::log::stack $stack_skip + + echo "Exiting with status ${code}" >&2 + exit "${code}" +} + find_files() { find . -not \( \ \( \ -wholename './_output' \ -o -wholename './_tools' \ -o -wholename './.*' \ - -o -wholename './pkg/assets/bindata.go' \ - -o -wholename './pkg/assets/*/bindata.go' \ - -o -wholename './openshift.local.*' \ -o -wholename '*/Godeps/*' \ \) -prune \ \) -name '*.go' | sort -u diff --git a/images/release/Dockerfile b/images/release/Dockerfile index 4400d33b6..2769f7c50 100644 --- a/images/release/Dockerfile +++ b/images/release/Dockerfile @@ -5,12 +5,7 @@ # FROM openshift/origin-base -# components from EPEL must be installed in a separate yum install step -RUN yum install -y git tar wget socat hostname epel-release && \ - yum clean all - -RUN yum install -y hg golang golang-pkg-darwin-amd64 golang-pkg-windows-amd64 && \ - yum clean all +RUN yum install -y zip hg golang golang-pkg-darwin-amd64 golang-pkg-windows-amd64 golang-pkg-linux-386 && yum clean all ENV GOPATH /go