Updated release procedure, added description of that to HACKING.md

AUTHORS

@@ -5,3 +5,4 @@ Ben Parees <bparees@redhat.com>
 Michal Fojtik <mfojtik@redhat.com>
 Dan McPherson <dmcphers@redhat.com>
 Cesar Wong <cewong@redhat.com>
+Maciej Szulik <maszulik@redhat.com>

HACKING.md

@@ -0,0 +1,83 @@
+Hacking on source-to-image
+==========================
+
+## Building a Release
+
+To build a STI release you run the `hack/build-release.sh` script on a system with Docker,
+which will create a build environment image and then execute a cross platform Go build within it. The build
+output will be copied to `_output/releases` as a set of tars containing each version.
+
+1. Create a new git tag `git tag v0.X.X -a -m "v0.X.X" HEAD`
+2. Push the tag to GitHub `git push origin --tags` where `origin` is `github.com/openshift/source-to-image.git`
+4. Run `hack/build-release.sh`
+5. Upload the binary artifacts generated by that build to GitHub release page.
+6. Send an email to the dev list, including the important changes prior to the release.
+
+## Test Suites
+
+STI uses two levels of testing - unit tests and integration tests.
+
+### Unit tests
+
+Unit tests follow standard Go conventions and are intended to test the behavior and output of a
+single package in isolation. All code is expected to be easily testable with mock interfaces and
+stubs, and when they are not it usually means that there's a missing interface or abstraction in the
+code. A unit test should focus on testing that branches and error conditions are properly returned
+and that the interface and code flows work as described. Unit tests can depend on other packages but
+should not depend on other components.
+
+The unit tests for an entire package should not take more than 0.5s to run, and if they do, are
+probably not really unit tests or need to be rewritten to avoid sleeps or pauses. Coverage on a unit
+test should be above 70% unless the units are a special case.
+
+Run the unit tests with:
+
+    $ hack/test-go.sh
+
+or an individual package unit test with:
+
+    $ hack/test-go.sh pkg/build/strategies/sti
+
+To run only a certain regex of tests in a package, use:
+
+    $ hack/test-go.sh pkg/build/strategies/sti -test.run=TestLayeredBuild
+
+To get verbose output add `-v` to the end:
+
+    $ hack/test-go.sh pkg/build/strategies/sti -test.run=TestLayeredBuild -v
+
+To run all tests with verbose output:
+
+    $ hack/test-go.sh "" -v
+
+To turn off or change the coverage mode, which is `-cover -covermode=atomic` by default, use:
+
+    $ STI_COVER="" hack/test-go.sh
+
+To run tests without the go race detector, which is on by default, use:
+
+    $ STI_RACE="" hack/test-go.sh
+
+A line coverage report is run by default when testing a single package.
+To create a coverage report for all packages:
+
+    $ OUTPUT_COVERAGE=true hack/test-go.sh pkg/build/strategies/sti
+
+### Integration tests
+
+The second category are integration tests which verify the whole STI flow.
+
+Run the integration tests with:
+
+    $ hack/test-integration.sh
+
+## Installing Godep
+
+STI uses [Godep](https://github.com/tools/godep) for dependency management.
+Godep allows versions of dependent packages to be locked at a specific commit by *vendoring* them
+(checking a copy of them into `Godeps/_workspace/`).  This means that everything you need for
+STI is checked into this repository.  To install `godep` locally run:
+
+    $ go get github.com/tools/godep
+
+If you are not updating packages you should not need godep installed.

hack/build-cross.sh

@@ -9,11 +9,18 @@ set -o pipefail
 STI_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${STI_ROOT}/hack/common.sh"
 
-STI_BUILD_PLATFORMS=("${STI_COMPILE_PLATFORMS[@]-}")
-sti::build::build_binaries "${STI_COMPILE_TARGETS[@]-}"
-
+# Build the primary for all platforms
 STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}")
 sti::build::build_binaries "${STI_CROSS_COMPILE_TARGETS[@]}"
 
-STI_RELEASE_ARCHIVES="${STI_OUTPUT}/releases"
+# Build image binaries for a subset of platforms. Image binaries are currently
+# linux-only, and are compiled with flags to make them static for use in Docker
+# images "FROM scratch".
+STI_BUILD_PLATFORMS=("${STI_IMAGE_COMPILE_PLATFORMS[@]-}")
+CGO_ENABLED=0 STI_GOFLAGS="-a" sti::build::build_binaries "${STI_IMAGE_COMPILE_TARGETS[@]-}"
+
+# Make the primary release.
+STI_RELEASE_ARCHIVE="source-to-image"
+STI_RELEASE_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}")
+STI_RELEASE_BINARIES=("${STI_CROSS_COMPILE_BINARIES[@]}")
 sti::build::place_bins

hack/build-release.sh

@@ -13,34 +13,35 @@ source "${STI_ROOT}/hack/common.sh"
 # Go to the top of the tree.
 cd "${STI_ROOT}"
 
+# Build the images
+echo "++ Building openshift/sti-release"
+docker build -q --tag openshift/sti-release "${STI_ROOT}/images/release"
+
 context="${STI_ROOT}/_output/buildenv-context"
 
-# clean existing output
+# Clean existing output.
 rm -rf "${STI_ROOT}/_output/local/releases"
 rm -rf "${STI_ROOT}/_output/local/go/bin"
 rm -rf "${context}"
 mkdir -p "${context}"
 mkdir -p "${STI_ROOT}/_output/local"
 
-# generate version definitions
+# Generate version definitions.
 sti::build::get_version_vars
 sti::build::save_version_vars "${context}/sti-version-defs"
 
-# create the input archive
+# Create the input archive.
 git archive --format=tar -o "${context}/archive.tar" HEAD
 tar -rf "${context}/archive.tar" -C "${context}" sti-version-defs
 gzip -f "${context}/archive.tar"
 
-# build in the clean environment
+# Perform the build and release in Docker.
 cat "${context}/archive.tar.gz" | docker run -i --cidfile="${context}/cid" openshift/sti-release
 docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_ROOT}/_output/local"
+echo "${STI_GIT_COMMIT}" > "${STI_ROOT}/_output/local/releases/.commit"
 
-# copy the linux release back to the _output/go/bin dir
-releases=$(find _output/local/releases/ -print | grep 'source-to-image-.*-linux-' --color=never)
-if [[ $(echo $releases | wc -l) -ne 1 ]]; then
-  echo "There should be exactly one Linux release tar in _output/local/releases"
-  exit 1
-fi
-bindir="_output/local/go/bin"
-mkdir -p "${bindir}"
-tar mxzf "${releases}" -C "${bindir}"
+# Copy the linux release archives release back to the local _output/local/go/bin directory.
+sti::build::detect_local_release_tars "linux"
+
+mkdir -p "${STI_LOCAL_BINPATH}"
+tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_LOCAL_BINPATH}"

hack/common.sh

@@ -18,16 +18,12 @@ STI_ROOT=$(
 STI_OUTPUT_SUBPATH="${STI_OUTPUT_SUBPATH:-_output/local}"
 STI_OUTPUT="${STI_ROOT}/${STI_OUTPUT_SUBPATH}"
 STI_OUTPUT_BINPATH="${STI_OUTPUT}/bin"
+STI_LOCAL_BINPATH="${STI_ROOT}/_output/local/go/bin"
+STI_LOCAL_RELEASEPATH="${STI_ROOT}/_output/local/releases"
 
 readonly STI_GO_PACKAGE=github.com/openshift/source-to-image
 readonly STI_GOPATH="${STI_OUTPUT}/go"
 
-readonly STI_COMPILE_PLATFORMS=(
-  linux/amd64
-)
-readonly STI_COMPILE_TARGETS=(
-)
-
 readonly STI_CROSS_COMPILE_PLATFORMS=(
   linux/amd64
   darwin/amd64
@@ -36,10 +32,9 @@ readonly STI_CROSS_COMPILE_PLATFORMS=(
 readonly STI_CROSS_COMPILE_TARGETS=(
   cmd/sti
 )
-# readonly STI_CROSS_COMPILE_BINARIES=("${STI_CROSS_COMPILE_TARGETS[@]##*/}")
+readonly STI_CROSS_COMPILE_BINARIES=("${STI_CROSS_COMPILE_TARGETS[@]##*/}")
 
 readonly STI_ALL_TARGETS=(
-  "${STI_COMPILE_TARGETS[@]-}"
   "${STI_CROSS_COMPILE_TARGETS[@]}"
 )
 # readonly STI_ALL_BINARIES=("${STI_ALL_TARGETS[@]##*/}")
@@ -196,8 +191,8 @@ EOF
 # This will take binaries from $GOPATH/bin and copy them to the appropriate
 # place in ${STI_OUTPUT_BINDIR}
 #
-# If STI_RELEASE_ARCHIVES is set to a directory, it will have tar archives of
-# each CROSS_COMPILE_PLATFORM created
+# If STI_RELEASE_ARCHIVE is set to a directory, it will have tar archives of
+# each STI_RELEASE_PLATFORMS created
 #
 # Ideally this wouldn't be necessary and we could just set GOBIN to
 # STI_OUTPUT_BINDIR but that won't work in the face of cross compilation.  'go
@@ -211,14 +206,12 @@ sti::build::place_bins() {
 
     echo "++ Placing binaries"
 
-    if [[ "${STI_RELEASE_ARCHIVES-}" != "" ]]; then
+    if [[ "${STI_RELEASE_ARCHIVE-}" != "" ]]; then
       sti::build::get_version_vars
-      rm -rf "${STI_RELEASE_ARCHIVES}"
-      mkdir -p "${STI_RELEASE_ARCHIVES}"
+      mkdir -p "${STI_LOCAL_RELEASEPATH}"
     fi
 
-    local platform
-    for platform in "${STI_CROSS_COMPILE_PLATFORMS[@]}"; do
+    for platform in "${STI_RELEASE_PLATFORMS[@]-(host_platform)}"; do
       # The substitution on platform_src below will replace all slashes with
       # underscores.  It'll transform darwin/amd64 -> darwin_amd64.
       local platform_src="/${platform//\//_}"
@@ -226,23 +219,74 @@ sti::build::place_bins() {
         platform_src=""
       fi
 
+      # Skip this directory if the platform has no binaries.
       local full_binpath_src="${STI_GOPATH}/bin${platform_src}"
-      if [[ -d "${full_binpath_src}" ]]; then
-        mkdir -p "${STI_OUTPUT_BINPATH}/${platform}"
-        find "${full_binpath_src}" -maxdepth 1 -type f -exec \
-          rsync -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \;
-
-        if [[ "${STI_RELEASE_ARCHIVES-}" != "" ]]; then
-          local platform_segment="${platform//\//-}"
-          local archive_name="source-to-image-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz"
-          echo "++ Creating ${archive_name}"
-          tar -czf "${STI_RELEASE_ARCHIVES}/${archive_name}" -C "${STI_OUTPUT_BINPATH}/${platform}" .
-        fi
+      if [[ ! -d "${full_binpath_src}" ]]; then
+        continue
       fi
+
+      mkdir -p "${STI_OUTPUT_BINPATH}/${platform}"
+
+      # Create an array of binaries to release. Append .exe variants if the platform is windows.
+      local -a binaries=()
+      local binary
+      for binary in "${STI_RELEASE_BINARIES[@]}"; do
+        binaries+=("${binary}")
+        if [[ $platform == "windows/amd64" ]]; then
+          binaries+=("${binary}.exe")
+        fi
+      done
+
+      # Copy only the specified release binaries to the shared STI_OUTPUT_BINPATH.
+      local -a includes=()
+      for binary in "${binaries[@]}"; do
+        includes+=("--include=${binary}")
+      done
+      find "${full_binpath_src}" -maxdepth 1 -type f -exec \
+        rsync "${includes[@]}" --exclude="*" -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \;
+
+      # If no release archive was requested, we're done.
+      if [[ "${STI_RELEASE_ARCHIVE-}" == "" ]]; then
+        continue
+      fi
+
+      # Create a temporary bin directory containing only the binaries marked for release.
+      local release_binpath=$(mktemp -d sti.release.${STI_RELEASE_ARCHIVE}.XXX)
+      find "${full_binpath_src}" -maxdepth 1 -type f -exec \
+        rsync "${includes[@]}" --exclude="*" -pt {} "${release_binpath}" \;
+
+      # Create the release archive.
+      local platform_segment="${platform//\//-}"
+      local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz"
+
+      echo "++ Creating ${archive_name}"
+      tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" .
+      rm -rf "${release_binpath}"
     done
   )
 }
 
+# sti::build::detect_local_release_tars verifies there is only one primary and one
+# image binaries release tar in STI_LOCAL_RELEASEPATH for the given platform specified by
+# argument 1, exiting if more than one of either is found.
+#
+# If the tars are discovered, their full paths are exported to the following env vars:
+#
+#   STI_PRIMARY_RELEASE_TAR
+sti::build::detect_local_release_tars() {
+  local platform="$1"
+
+  local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}-*)
+  if [[ $(echo "${primary}" | wc -l) -ne 1 ]]; then
+    echo "There should be exactly one ${platform} primary tar in $STI_LOCAL_RELEASEPATH"
+    exit 2
+  fi
+
+  export STI_PRIMARY_RELEASE_TAR="${primary}"
+  export STI_RELEASE_COMMIT="$(cat ${STI_LOCAL_RELEASEPATH}/.commit)"
+}
+
+
 # sti::build::get_version_vars loads the standard version variables as
 # ENV vars
 sti::build::get_version_vars() {