Merge pull request #331 from soltysh/update_build

Merged by openshift-bot

CONTRIBUTING.md

@@ -93,12 +93,12 @@ If you are not updating packages you should not need godep installed.
 
 ## Building a Release
 
-To build a S2I release you run the `hack/build-release.sh` script on a system with Docker,
+To build a S2I release you run `make release` on a system with Docker,
 which will create a build environment image and then execute a cross platform Go build within it. The build
 output will be copied to `_output/releases` as a set of tars containing each version.
 
 1. Create a new git tag `git tag vX.X.X -a -m "vX.X.X" HEAD`
 2. Push the tag to GitHub `git push origin --tags` where `origin` is `github.com/openshift/source-to-image.git`
-4. Run `hack/build-release.sh`
+4. Run `make release`
 5. Upload the binary artifacts generated by that build to GitHub release page.
 6. Send an email to the dev list, including the important changes prior to the release.

Makefile

@@ -6,6 +6,7 @@
 #   check: Run tests.
 #   test: Run tests.
 #   clean: Clean up.
+#   release: Build release.
 
 OUT_DIR = _output
 OUT_PKG_DIR = Godeps/_workspace/pkg
@@ -24,7 +25,6 @@ all build:
 	hack/build-go.sh
 .PHONY: all build
 
-
 # Verify code is properly organized.
 #
 # Example:
@@ -35,7 +35,6 @@ verify: build
 	hack/verify-govet.sh || true
 .PHONY: verify
 
-
 # Install travis dependencies
 #
 # Example:
@@ -44,21 +43,22 @@ install-travis:
 	hack/install-tools.sh
 .PHONY: install-travis
 
-# Build and run tests.
+# Build and run unit tests
 #
 # Args:
 #   WHAT: Directory names to test.  All *_test.go files under these
 #     directories will be run.  If not specified, "everything" will be tested.
 #   TESTS: Same as WHAT.
 #   GOFLAGS: Extra flags to pass to 'go' when building.
+#   TESTFLAGS: Extra flags that should only be passed to hack/test-go.sh
 #
 # Example:
 #   make check
 #   make test
 #   make check WHAT=pkg/build GOFLAGS=-v
-check test: verify
-	STI_COVER="-cover -covermode=atomic" STI_RACE="-race" hack/test-go.sh $(WHAT) $(TESTS)
-.PHONY: check test
+check:
+	hack/test-go.sh $(WHAT) $(TESTS) $(TESTFLAGS)
+.PHONY: check
 
 # Remove all build artifacts.
 #
@@ -67,3 +67,12 @@ check test: verify
 clean:
 	rm -rf $(OUT_DIR) $(OUT_PKG_DIR)
 .PHONY: clean
+
+# Build the release.
+#
+# Example:
+#   make release
+release: clean
+	hack/build-release.sh
+	hack/extract-release.sh
+.PHONY: release

hack/build-cross.sh

@@ -6,21 +6,19 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+STARTTIME=$(date +%s)
 STI_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${STI_ROOT}/hack/common.sh"
+source "${STI_ROOT}/hack/util.sh"
+sti::log::install_errexit
 
 # Build the primary for all platforms
 STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}")
 sti::build::build_binaries "${STI_CROSS_COMPILE_TARGETS[@]}"
 
-# Build image binaries for a subset of platforms. Image binaries are currently
-# linux-only, and are compiled with flags to make them static for use in Docker
-# images "FROM scratch".
-STI_BUILD_PLATFORMS=("${STI_IMAGE_COMPILE_PLATFORMS[@]-}")
-CGO_ENABLED=0 STI_GOFLAGS="-a" sti::build::build_binaries "${STI_IMAGE_COMPILE_TARGETS[@]-}"
-
 # Make the primary release.
 STI_RELEASE_ARCHIVE="source-to-image"
-STI_RELEASE_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}")
-STI_RELEASE_BINARIES=("${STI_CROSS_COMPILE_BINARIES[@]}")
-sti::build::place_bins
+STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}")
+sti::build::place_bins "${STI_CROSS_COMPILE_BINARIES[@]}"
+
+ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"

hack/build-release.sh

@@ -7,8 +7,11 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+STARTTIME=$(date +%s)
 STI_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${STI_ROOT}/hack/common.sh"
+source "${STI_ROOT}/hack/util.sh"
+sti::log::install_errexit
 
 # Go to the top of the tree.
 cd "${STI_ROOT}"
@@ -20,30 +23,26 @@ docker build -q --tag openshift/sti-release "${STI_ROOT}/images/release"
 context="${STI_ROOT}/_output/buildenv-context"
 
 # Clean existing output.
-rm -rf "${STI_ROOT}/_output/local/releases"
-rm -rf "${STI_ROOT}/_output/local/go/bin"
+rm -rf "${STI_LOCAL_RELEASEPATH}"
 rm -rf "${context}"
 mkdir -p "${context}"
-mkdir -p "${STI_ROOT}/_output/local"
+mkdir -p "${STI_OUTPUT}"
 
 # Generate version definitions.
+# You can commit a specific version by specifying STI_GIT_COMMIT="" prior to build
 sti::build::get_version_vars
 sti::build::save_version_vars "${context}/sti-version-defs"
 
+echo "++ Building release ${STI_GIT_VERSION}"
+
 # Create the input archive.
-git archive --format=tar -o "${context}/archive.tar" HEAD
+git archive --format=tar -o "${context}/archive.tar" "${STI_GIT_COMMIT}"
 tar -rf "${context}/archive.tar" -C "${context}" sti-version-defs
 gzip -f "${context}/archive.tar"
 
 # Perform the build and release in Docker.
 cat "${context}/archive.tar.gz" | docker run -i --cidfile="${context}/cid" openshift/sti-release
-docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_ROOT}/_output/local"
-echo "${STI_GIT_COMMIT}" > "${STI_ROOT}/_output/local/releases/.commit"
+docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_OUTPUT}"
+echo "${STI_GIT_COMMIT}" > "${STI_LOCAL_RELEASEPATH}/.commit"
 
-# Copy the linux release archives release back to the local _output/local/go/bin directory.
-sti::build::detect_local_release_tars "linux"
-
-mkdir -p "${STI_LOCAL_BINPATH}"
-tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_LOCAL_BINPATH}"
-
-sti::build::make_binary_symlinks
+ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"

hack/common.sh

@@ -18,8 +18,8 @@ STI_ROOT=$(
 STI_OUTPUT_SUBPATH="${STI_OUTPUT_SUBPATH:-_output/local}"
 STI_OUTPUT="${STI_ROOT}/${STI_OUTPUT_SUBPATH}"
 STI_OUTPUT_BINPATH="${STI_OUTPUT}/bin"
-STI_LOCAL_BINPATH="${STI_ROOT}/_output/local/go/bin"
-STI_LOCAL_RELEASEPATH="${STI_ROOT}/_output/local/releases"
+STI_LOCAL_BINPATH="${STI_OUTPUT}/go/bin"
+STI_LOCAL_RELEASEPATH="${STI_OUTPUT}/releases"
 
 readonly STI_GO_PACKAGE=github.com/openshift/source-to-image
 readonly STI_GOPATH="${STI_OUTPUT}/go"
@@ -28,6 +28,7 @@ readonly STI_CROSS_COMPILE_PLATFORMS=(
   linux/amd64
   darwin/amd64
   windows/amd64
+  linux/386
 )
 readonly STI_CROSS_COMPILE_TARGETS=(
   cmd/s2i
@@ -44,6 +45,10 @@ readonly STI_BINARY_SYMLINKS=(
 readonly STI_BINARY_COPY=(
   sti
 )
+readonly STI_BINARY_RELEASE_WINDOWS=(
+  sti.exe
+  s2i.exe
+)
 
 # sti::build::binaries_from_targets take a list of build targets and return the
 # full go package to be built
@@ -78,32 +83,7 @@ sti::build::build_binaries() {
     local version_ldflags
     version_ldflags=$(sti::build::ldflags)
 
-    # Use eval to preserve embedded quoted strings.
-    local goflags
-    eval "goflags=(${STI_GOFLAGS:-})"
-
-    local -a targets=()
-    local arg
-    for arg; do
-      if [[ "${arg}" == -* ]]; then
-        # Assume arguments starting with a dash are flags to pass to go.
-        goflags+=("${arg}")
-      else
-        targets+=("${arg}")
-      fi
-    done
-
-    if [[ ${#targets[@]} -eq 0 ]]; then
-      targets=("${STI_ALL_TARGETS[@]}")
-    fi
-
-    local -a platforms=("${STI_BUILD_PLATFORMS[@]:+${STI_BUILD_PLATFORMS[@]}}")
-    if [[ ${#platforms[@]} -eq 0 ]]; then
-      platforms=("$(sti::build::host_platform)")
-    fi
-
-    local binaries
-    binaries=($(sti::build::binaries_from_targets "${targets[@]}"))
+    sti::build::export_targets "$@"
 
     local platform
     for platform in "${platforms[@]}"; do
@@ -117,6 +97,38 @@ sti::build::build_binaries() {
   )
 }
 
+# Generates the set of target packages, binaries, and platforms to build for.
+# Accepts binaries via $@, and platforms via STI_BUILD_PLATFORMS, or defaults to
+# the current platform.
+sti::build::export_targets() {
+  # Use eval to preserve embedded quoted strings.
+  local goflags
+  eval "goflags=(${STI_GOFLAGS:-})"
+
+  targets=()
+  local arg
+  for arg; do
+    if [[ "${arg}" == -* ]]; then
+      # Assume arguments starting with a dash are flags to pass to go.
+      goflags+=("${arg}")
+    else
+      targets+=("${arg}")
+    fi
+  done
+
+  if [[ ${#targets[@]} -eq 0 ]]; then
+    targets=("${STI_ALL_TARGETS[@]}")
+  fi
+
+  binaries=($(sti::build::binaries_from_targets "${targets[@]}"))
+
+  platforms=("${STI_BUILD_PLATFORMS[@]:+${STI_BUILD_PLATFORMS[@]}}")
+  if [[ ${#platforms[@]} -eq 0 ]]; then
+    platforms=("$(sti::build::host_platform)")
+  fi
+}
+
+
 # Takes the platform name ($1) and sets the appropriate golang env variables
 # for that platform.
 sti::build::set_platform_envs() {
@@ -167,7 +179,7 @@ sti::build::setup_env() {
   sti::build::create_gopath_tree
 
   if [[ -z "$(which go)" ]]; then
-    echo <<EOF
+    cat <<EOF
 
 Can't find 'go' in PATH, please fix and retry.
 See http://golang.org/doc/install for installation instructions.
@@ -176,6 +188,24 @@ EOF
     exit 2
   fi
 
+  # Travis continuous build uses a head go release that doesn't report
+  # a version number, so we skip this check on Travis.  It's unnecessary
+  # there anyway.
+  if [[ "${TRAVIS:-}" != "true" ]]; then
+    local go_version
+    go_version=($(go version))
+    if [[ "${go_version[2]}" < "go1.4" ]]; then
+      cat <<EOF
+
+Detected go version: ${go_version[*]}.
+S2I requires go version 1.4 or greater.
+Please install Go version 1.4 or later.
+
+EOF
+      exit 2
+    fi
+  fi
+
   GOPATH=${STI_GOPATH}
 
   # Append STI_EXTRA_GOPATH to the GOPATH if it is defined.
@@ -217,7 +247,9 @@ sti::build::place_bins() {
       mkdir -p "${STI_LOCAL_RELEASEPATH}"
     fi
 
-    for platform in "${STI_RELEASE_PLATFORMS[@]-(host_platform)}"; do
+    sti::build::export_targets "$@"
+
+    for platform in "${platforms[@]}"; do
       # The substitution on platform_src below will replace all slashes with
       # underscores.  It'll transform darwin/amd64 -> darwin_amd64.
       local platform_src="/${platform//\//_}"
@@ -235,21 +267,19 @@ sti::build::place_bins() {
 
       # Create an array of binaries to release. Append .exe variants if the platform is windows.
       local -a binaries=()
-      local binary
-      for binary in "${STI_RELEASE_BINARIES[@]}"; do
-        binaries+=("${binary}")
+      for binary in "${targets[@]}"; do
+        binary=$(basename $binary)
         if [[ $platform == "windows/amd64" ]]; then
           binaries+=("${binary}.exe")
+        else
+          binaries+=("${binary}")
         fi
       done
 
-      # Copy only the specified release binaries to the shared STI_OUTPUT_BINPATH.
-      local -a includes=()
+      # Move the specified release binaries to the shared STI_OUTPUT_BINPATH.
       for binary in "${binaries[@]}"; do
-        includes+=("--include=${binary}")
+        mv "${full_binpath_src}/${binary}" "${STI_OUTPUT_BINPATH}/${platform}/"
       done
-      find "${full_binpath_src}" -maxdepth 1 -type f -exec \
-        rsync "${includes[@]}" --exclude="*" -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \;
 
       # If no release archive was requested, we're done.
       if [[ "${STI_RELEASE_ARCHIVE-}" == "" ]]; then
@@ -258,8 +288,9 @@ sti::build::place_bins() {
 
       # Create a temporary bin directory containing only the binaries marked for release.
       local release_binpath=$(mktemp -d sti.release.${STI_RELEASE_ARCHIVE}.XXX)
-      find "${full_binpath_src}" -maxdepth 1 -type f -exec \
-        rsync "${includes[@]}" --exclude="*" -pt {} "${release_binpath}" \;
+      for binary in "${binaries[@]}"; do
+        cp "${STI_OUTPUT_BINPATH}/${platform}/${binary}" "${release_binpath}/"
+      done
 
       # Create binary copies where specified.
       local suffix=""
@@ -275,10 +306,17 @@ sti::build::place_bins() {
 
       # Create the release archive.
       local platform_segment="${platform//\//-}"
-      local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz"
-
-      echo "++ Creating ${archive_name}"
-      tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" .
+      if [[ $platform == "windows/amd64" ]]; then
+        local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.zip"
+        echo "++ Creating ${archive_name}"
+        for file in "${STI_BINARY_RELEASE_WINDOWS[@]}"; do
+          zip "${STI_LOCAL_RELEASEPATH}/${archive_name}" -qj "${release_binpath}/${file}"
+        done
+      else
+        local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz"
+        echo "++ Creating ${archive_name}"
+        tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" .
+      fi
       rm -rf "${release_binpath}"
     done
   )
@@ -287,15 +325,13 @@ sti::build::place_bins() {
 # sti::build::make_binary_symlinks makes symlinks for the sti
 # binary in _output/local/go/bin
 sti::build::make_binary_symlinks() {
-  local host_platform
-  host_platform=$(sti::build::host_platform)
-
-  if [[ -f "${STI_LOCAL_BINPATH}/s2i" ]]; then
+  platform=$(sti::build::host_platform)
+  if [[ -f "${STI_OUTPUT_BINPATH}/${platform}/s2i" ]]; then
     for linkname in "${STI_BINARY_SYMLINKS[@]}"; do
-      if [[ $host_platform == "windows/amd64" ]]; then
-        cp "${STI_LOCAL_BINPATH}/s2i.exe" "${STI_LOCAL_BINPATH}/${linkname}.exe"
+      if [[ $platform == "windows/amd64" ]]; then
+        cp s2i "${STI_OUTPUT_BINPATH}/${platform}/${linkname}.exe"
       else
-        ln -sf "${STI_LOCAL_BINPATH}/s2i" "${STI_LOCAL_BINPATH}/${linkname}"
+        ln -sf s2i "${STI_OUTPUT_BINPATH}/${platform}/${linkname}"
       fi
     done
   fi
@@ -311,7 +347,15 @@ sti::build::make_binary_symlinks() {
 sti::build::detect_local_release_tars() {
   local platform="$1"
 
-  local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}-*)
+  if [[ ! -d "${STI_LOCAL_RELEASEPATH}" ]]; then
+    echo "There are no release artifacts in ${STI_LOCAL_RELEASEPATH}"
+    exit 2
+  fi
+  if [[ ! -f "${STI_LOCAL_RELEASEPATH}/.commit" ]]; then
+    echo "There is no release .commit identifier ${STI_LOCAL_RELEASEPATH}"
+    exit 2
+  fi
+  local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}*)
   if [[ $(echo "${primary}" | wc -l) -ne 1 ]]; then
     echo "There should be exactly one ${platform} primary tar in $STI_LOCAL_RELEASEPATH"
     exit 2

hack/extract-release.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# This script extracts a valid release tar into _output/releases. It requires hack/build-release.sh
+# to have been executed
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+STI_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${STI_ROOT}/hack/common.sh"
+
+# Go to the top of the tree.
+cd "${STI_ROOT}"
+
+# Copy the linux release archives release back to the local _output/local/bin/linux/amd64 directory.
+# TODO: support different OS's?
+sti::build::detect_local_release_tars "linux-amd64"
+
+mkdir -p "${STI_OUTPUT_BINPATH}/linux/amd64"
+tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_OUTPUT_BINPATH}/linux/amd64"
+
+sti::build::make_binary_symlinks

hack/test-stirunimage.sh

@@ -35,8 +35,8 @@ fi
 
 trap cleanup EXIT SIGINT
 
-echo "Running 'sti build --run=true ...'"
-s2i build git://github.com/bparees/openshift-jee-sample openshift/wildfly-8-centos test-jee-app --run=true &> "${STI_ROOT}/hack/sti-run.log" &
+echo "Running 's2i build --run=true ...'"
+s2i build git://github.com/bparees/openshift-jee-sample openshift/wildfly-81-centos7 test-jee-app --run=true &> "${STI_ROOT}/hack/sti-run.log" &
 export STI_PID=$!
 TIME_SEC=1000
 TIME_MIN=$((60 * $TIME_SEC))
@@ -48,8 +48,8 @@ set +e
 while [[ $(time_now) -lt $expire ]]; do
     grep  "as a result of the --run=true option" "${STI_ROOT}/hack/sti-run.log"
     if [ $? -eq 0 ]; then
-      echo "[INFO] Success running command sti --run=true"
-      
+      echo "[INFO] Success running command s2i --run=true"
+
       # use sigint so that sti post processing will remove docker container
       kill -2 "${STI_PID}"
       export KILLDONE="killed"
@@ -68,7 +68,7 @@ while [[ $(time_now) -lt $expire ]]; do
     sleep 1
 done
 
-echo "[INFO] Problem with sti --run=true, dumping ${STI_ROOT}/hack/sti-run.log"
+echo "[INFO] Problem with s2i --run=true, dumping ${STI_ROOT}/hack/sti-run.log"
 cat "${STI_ROOT}/hack/sti-run.log"
 set -e
 exit 1

hack/util.sh

@@ -2,15 +2,81 @@
 
 # Provides simple utility functions
 
+# Handler for when we exit automatically on an error.
+# Borrowed from https://gist.github.com/ahendrix/7030300
+sti::log::errexit() {
+  local err="${PIPESTATUS[@]}"
+
+  # If the shell we are in doesn't have errexit set (common in subshells) then
+  # don't dump stacks.
+  set +o | grep -qe "-o errexit" || return
+
+  set +o xtrace
+  local code="${1:-1}"
+  sti::log::error_exit "'${BASH_COMMAND}' exited with status $err" "${1:-1}" 1
+}
+
+sti::log::install_errexit() {
+  # trap ERR to provide an error handler whenever a command exits nonzero this
+  # is a more verbose version of set -o errexit
+  trap 'sti::log::errexit' ERR
+
+  # setting errtrace allows our ERR trap handler to be propagated to functions,
+  # expansions and subshells
+  set -o errtrace
+}
+
+# Print out the stack trace
+#
+# Args:
+#  $1 The number of stack frames to skip when printing.
+sti::log::stack() {
+  local stack_skip=${1:-0}
+  stack_skip=$((stack_skip + 1))
+  if [[ ${#FUNCNAME[@]} -gt $stack_skip ]]; then
+  echo "Call stack:" >&2
+  local i
+  for ((i=1 ; i <= ${#FUNCNAME[@]} - $stack_skip ; i++))
+  do
+    local frame_no=$((i - 1 + stack_skip))
+    local source_file=${BASH_SOURCE[$frame_no]}
+    local source_lineno=${BASH_LINENO[$((frame_no - 1))]}
+    local funcname=${FUNCNAME[$frame_no]}
+    echo "  $i: ${source_file}:${source_lineno} ${funcname}(...)" >&2
+  done
+  fi
+}
+
+# Log an error and exit.
+# Args:
+#  $1 Message to log with the error
+#  $2 The error code to return
+#  $3 The number of stack frames to skip when printing.
+sti::log::error_exit() {
+  local message="${1:-}"
+  local code="${2:-1}"
+  local stack_skip="${3:-0}"
+  stack_skip=$((stack_skip + 1))
+
+  local source_file=${BASH_SOURCE[$stack_skip]}
+  local source_line=${BASH_LINENO[$((stack_skip - 1))]}
+  echo "!!! Error in ${source_file}:${source_line}" >&2
+  [[ -z ${1-} ]] || {
+  echo "  ${1}" >&2
+  }
+
+  sti::log::stack $stack_skip
+
+  echo "Exiting with status ${code}" >&2
+  exit "${code}"
+}
+
 find_files() {
   find . -not \( \
       \( \
         -wholename './_output' \
         -o -wholename './_tools' \
         -o -wholename './.*' \
-        -o -wholename './pkg/assets/bindata.go' \
-        -o -wholename './pkg/assets/*/bindata.go' \
-        -o -wholename './openshift.local.*' \
         -o -wholename '*/Godeps/*' \
       \) -prune \
     \) -name '*.go' | sort -u

images/release/Dockerfile

@@ -5,12 +5,7 @@
 #
 FROM openshift/origin-base
 
-# components from EPEL must be installed in a separate yum install step
-RUN yum install -y git tar wget socat hostname epel-release && \
-    yum clean all
-
-RUN yum install -y hg golang golang-pkg-darwin-amd64 golang-pkg-windows-amd64 && \
-    yum clean all
+RUN yum install -y zip hg golang golang-pkg-darwin-amd64 golang-pkg-windows-amd64 golang-pkg-linux-386 && yum clean all
 
 ENV GOPATH /go