From b35986fc607c58bc42fe3dc336e5ad2ccdd1cd6b Mon Sep 17 00:00:00 2001 From: Maciej Szulik Date: Fri, 27 Feb 2015 13:43:05 +0100 Subject: [PATCH] Updated release procedure, added description of that to HACKING.md --- AUTHORS | 1 + HACKING.md | 83 +++++++++++++++++++++++++++++++++++++ hack/build-cross.sh | 15 +++++-- hack/build-release.sh | 27 ++++++------ hack/common.sh | 96 +++++++++++++++++++++++++++++++------------ 5 files changed, 179 insertions(+), 43 deletions(-) create mode 100644 HACKING.md diff --git a/AUTHORS b/AUTHORS index 5e20775df..9b4b2e9a1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,3 +5,4 @@ Ben Parees Michal Fojtik Dan McPherson Cesar Wong +Maciej Szulik diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 000000000..1a2fa38fa --- /dev/null +++ b/HACKING.md @@ -0,0 +1,83 @@ +Hacking on source-to-image +========================== + +## Building a Release + +To build a STI release you run the `hack/build-release.sh` script on a system with Docker, +which will create a build environment image and then execute a cross platform Go build within it. The build +output will be copied to `_output/releases` as a set of tars containing each version. + +1. Create a new git tag `git tag v0.X.X -a -m "v0.X.X" HEAD` +2. Push the tag to GitHub `git push origin --tags` where `origin` is `github.com/openshift/source-to-image.git` +4. Run `hack/build-release.sh` +5. Upload the binary artifacts generated by that build to GitHub release page. +6. Send an email to the dev list, including the important changes prior to the release. + +## Test Suites + +STI uses two levels of testing - unit tests and integration tests. + +### Unit tests + +Unit tests follow standard Go conventions and are intended to test the behavior and output of a +single package in isolation. All code is expected to be easily testable with mock interfaces and +stubs, and when they are not it usually means that there's a missing interface or abstraction in the +code. A unit test should focus on testing that branches and error conditions are properly returned +and that the interface and code flows work as described. Unit tests can depend on other packages but +should not depend on other components. + +The unit tests for an entire package should not take more than 0.5s to run, and if they do, are +probably not really unit tests or need to be rewritten to avoid sleeps or pauses. Coverage on a unit +test should be above 70% unless the units are a special case. + +Run the unit tests with: + + $ hack/test-go.sh + +or an individual package unit test with: + + $ hack/test-go.sh pkg/build/strategies/sti + +To run only a certain regex of tests in a package, use: + + $ hack/test-go.sh pkg/build/strategies/sti -test.run=TestLayeredBuild + +To get verbose output add `-v` to the end: + + $ hack/test-go.sh pkg/build/strategies/sti -test.run=TestLayeredBuild -v + +To run all tests with verbose output: + + $ hack/test-go.sh "" -v + +To turn off or change the coverage mode, which is `-cover -covermode=atomic` by default, use: + + $ STI_COVER="" hack/test-go.sh + +To run tests without the go race detector, which is on by default, use: + + $ STI_RACE="" hack/test-go.sh + +A line coverage report is run by default when testing a single package. +To create a coverage report for all packages: + + $ OUTPUT_COVERAGE=true hack/test-go.sh pkg/build/strategies/sti + +### Integration tests + +The second category are integration tests which verify the whole STI flow. + +Run the integration tests with: + + $ hack/test-integration.sh + +## Installing Godep + +STI uses [Godep](https://github.com/tools/godep) for dependency management. +Godep allows versions of dependent packages to be locked at a specific commit by *vendoring* them +(checking a copy of them into `Godeps/_workspace/`). This means that everything you need for +STI is checked into this repository. To install `godep` locally run: + + $ go get github.com/tools/godep + +If you are not updating packages you should not need godep installed. diff --git a/hack/build-cross.sh b/hack/build-cross.sh index 33622feb0..b59b6082a 100755 --- a/hack/build-cross.sh +++ b/hack/build-cross.sh @@ -9,11 +9,18 @@ set -o pipefail STI_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${STI_ROOT}/hack/common.sh" -STI_BUILD_PLATFORMS=("${STI_COMPILE_PLATFORMS[@]-}") -sti::build::build_binaries "${STI_COMPILE_TARGETS[@]-}" - +# Build the primary for all platforms STI_BUILD_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}") sti::build::build_binaries "${STI_CROSS_COMPILE_TARGETS[@]}" -STI_RELEASE_ARCHIVES="${STI_OUTPUT}/releases" +# Build image binaries for a subset of platforms. Image binaries are currently +# linux-only, and are compiled with flags to make them static for use in Docker +# images "FROM scratch". +STI_BUILD_PLATFORMS=("${STI_IMAGE_COMPILE_PLATFORMS[@]-}") +CGO_ENABLED=0 STI_GOFLAGS="-a" sti::build::build_binaries "${STI_IMAGE_COMPILE_TARGETS[@]-}" + +# Make the primary release. +STI_RELEASE_ARCHIVE="source-to-image" +STI_RELEASE_PLATFORMS=("${STI_CROSS_COMPILE_PLATFORMS[@]}") +STI_RELEASE_BINARIES=("${STI_CROSS_COMPILE_BINARIES[@]}") sti::build::place_bins diff --git a/hack/build-release.sh b/hack/build-release.sh index 73f5e9385..2629c550d 100755 --- a/hack/build-release.sh +++ b/hack/build-release.sh @@ -13,34 +13,35 @@ source "${STI_ROOT}/hack/common.sh" # Go to the top of the tree. cd "${STI_ROOT}" +# Build the images +echo "++ Building openshift/sti-release" +docker build -q --tag openshift/sti-release "${STI_ROOT}/images/release" + context="${STI_ROOT}/_output/buildenv-context" -# clean existing output +# Clean existing output. rm -rf "${STI_ROOT}/_output/local/releases" rm -rf "${STI_ROOT}/_output/local/go/bin" rm -rf "${context}" mkdir -p "${context}" mkdir -p "${STI_ROOT}/_output/local" -# generate version definitions +# Generate version definitions. sti::build::get_version_vars sti::build::save_version_vars "${context}/sti-version-defs" -# create the input archive +# Create the input archive. git archive --format=tar -o "${context}/archive.tar" HEAD tar -rf "${context}/archive.tar" -C "${context}" sti-version-defs gzip -f "${context}/archive.tar" -# build in the clean environment +# Perform the build and release in Docker. cat "${context}/archive.tar.gz" | docker run -i --cidfile="${context}/cid" openshift/sti-release docker cp $(cat ${context}/cid):/go/src/github.com/openshift/source-to-image/_output/local/releases "${STI_ROOT}/_output/local" +echo "${STI_GIT_COMMIT}" > "${STI_ROOT}/_output/local/releases/.commit" -# copy the linux release back to the _output/go/bin dir -releases=$(find _output/local/releases/ -print | grep 'source-to-image-.*-linux-' --color=never) -if [[ $(echo $releases | wc -l) -ne 1 ]]; then - echo "There should be exactly one Linux release tar in _output/local/releases" - exit 1 -fi -bindir="_output/local/go/bin" -mkdir -p "${bindir}" -tar mxzf "${releases}" -C "${bindir}" +# Copy the linux release archives release back to the local _output/local/go/bin directory. +sti::build::detect_local_release_tars "linux" + +mkdir -p "${STI_LOCAL_BINPATH}" +tar mxzf "${STI_PRIMARY_RELEASE_TAR}" -C "${STI_LOCAL_BINPATH}" diff --git a/hack/common.sh b/hack/common.sh index 1863a7aa1..491271c3d 100755 --- a/hack/common.sh +++ b/hack/common.sh @@ -18,16 +18,12 @@ STI_ROOT=$( STI_OUTPUT_SUBPATH="${STI_OUTPUT_SUBPATH:-_output/local}" STI_OUTPUT="${STI_ROOT}/${STI_OUTPUT_SUBPATH}" STI_OUTPUT_BINPATH="${STI_OUTPUT}/bin" +STI_LOCAL_BINPATH="${STI_ROOT}/_output/local/go/bin" +STI_LOCAL_RELEASEPATH="${STI_ROOT}/_output/local/releases" readonly STI_GO_PACKAGE=github.com/openshift/source-to-image readonly STI_GOPATH="${STI_OUTPUT}/go" -readonly STI_COMPILE_PLATFORMS=( - linux/amd64 -) -readonly STI_COMPILE_TARGETS=( -) - readonly STI_CROSS_COMPILE_PLATFORMS=( linux/amd64 darwin/amd64 @@ -36,10 +32,9 @@ readonly STI_CROSS_COMPILE_PLATFORMS=( readonly STI_CROSS_COMPILE_TARGETS=( cmd/sti ) -# readonly STI_CROSS_COMPILE_BINARIES=("${STI_CROSS_COMPILE_TARGETS[@]##*/}") +readonly STI_CROSS_COMPILE_BINARIES=("${STI_CROSS_COMPILE_TARGETS[@]##*/}") readonly STI_ALL_TARGETS=( - "${STI_COMPILE_TARGETS[@]-}" "${STI_CROSS_COMPILE_TARGETS[@]}" ) # readonly STI_ALL_BINARIES=("${STI_ALL_TARGETS[@]##*/}") @@ -196,8 +191,8 @@ EOF # This will take binaries from $GOPATH/bin and copy them to the appropriate # place in ${STI_OUTPUT_BINDIR} # -# If STI_RELEASE_ARCHIVES is set to a directory, it will have tar archives of -# each CROSS_COMPILE_PLATFORM created +# If STI_RELEASE_ARCHIVE is set to a directory, it will have tar archives of +# each STI_RELEASE_PLATFORMS created # # Ideally this wouldn't be necessary and we could just set GOBIN to # STI_OUTPUT_BINDIR but that won't work in the face of cross compilation. 'go @@ -211,14 +206,12 @@ sti::build::place_bins() { echo "++ Placing binaries" - if [[ "${STI_RELEASE_ARCHIVES-}" != "" ]]; then + if [[ "${STI_RELEASE_ARCHIVE-}" != "" ]]; then sti::build::get_version_vars - rm -rf "${STI_RELEASE_ARCHIVES}" - mkdir -p "${STI_RELEASE_ARCHIVES}" + mkdir -p "${STI_LOCAL_RELEASEPATH}" fi - local platform - for platform in "${STI_CROSS_COMPILE_PLATFORMS[@]}"; do + for platform in "${STI_RELEASE_PLATFORMS[@]-(host_platform)}"; do # The substitution on platform_src below will replace all slashes with # underscores. It'll transform darwin/amd64 -> darwin_amd64. local platform_src="/${platform//\//_}" @@ -226,23 +219,74 @@ sti::build::place_bins() { platform_src="" fi + # Skip this directory if the platform has no binaries. local full_binpath_src="${STI_GOPATH}/bin${platform_src}" - if [[ -d "${full_binpath_src}" ]]; then - mkdir -p "${STI_OUTPUT_BINPATH}/${platform}" - find "${full_binpath_src}" -maxdepth 1 -type f -exec \ - rsync -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \; - - if [[ "${STI_RELEASE_ARCHIVES-}" != "" ]]; then - local platform_segment="${platform//\//-}" - local archive_name="source-to-image-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz" - echo "++ Creating ${archive_name}" - tar -czf "${STI_RELEASE_ARCHIVES}/${archive_name}" -C "${STI_OUTPUT_BINPATH}/${platform}" . - fi + if [[ ! -d "${full_binpath_src}" ]]; then + continue fi + + mkdir -p "${STI_OUTPUT_BINPATH}/${platform}" + + # Create an array of binaries to release. Append .exe variants if the platform is windows. + local -a binaries=() + local binary + for binary in "${STI_RELEASE_BINARIES[@]}"; do + binaries+=("${binary}") + if [[ $platform == "windows/amd64" ]]; then + binaries+=("${binary}.exe") + fi + done + + # Copy only the specified release binaries to the shared STI_OUTPUT_BINPATH. + local -a includes=() + for binary in "${binaries[@]}"; do + includes+=("--include=${binary}") + done + find "${full_binpath_src}" -maxdepth 1 -type f -exec \ + rsync "${includes[@]}" --exclude="*" -pt {} "${STI_OUTPUT_BINPATH}/${platform}" \; + + # If no release archive was requested, we're done. + if [[ "${STI_RELEASE_ARCHIVE-}" == "" ]]; then + continue + fi + + # Create a temporary bin directory containing only the binaries marked for release. + local release_binpath=$(mktemp -d sti.release.${STI_RELEASE_ARCHIVE}.XXX) + find "${full_binpath_src}" -maxdepth 1 -type f -exec \ + rsync "${includes[@]}" --exclude="*" -pt {} "${release_binpath}" \; + + # Create the release archive. + local platform_segment="${platform//\//-}" + local archive_name="${STI_RELEASE_ARCHIVE}-${STI_GIT_VERSION}-${STI_GIT_COMMIT}-${platform_segment}.tar.gz" + + echo "++ Creating ${archive_name}" + tar -czf "${STI_LOCAL_RELEASEPATH}/${archive_name}" -C "${release_binpath}" . + rm -rf "${release_binpath}" done ) } +# sti::build::detect_local_release_tars verifies there is only one primary and one +# image binaries release tar in STI_LOCAL_RELEASEPATH for the given platform specified by +# argument 1, exiting if more than one of either is found. +# +# If the tars are discovered, their full paths are exported to the following env vars: +# +# STI_PRIMARY_RELEASE_TAR +sti::build::detect_local_release_tars() { + local platform="$1" + + local primary=$(find ${STI_LOCAL_RELEASEPATH} -maxdepth 1 -type f -name source-to-image-*-${platform}-*) + if [[ $(echo "${primary}" | wc -l) -ne 1 ]]; then + echo "There should be exactly one ${platform} primary tar in $STI_LOCAL_RELEASEPATH" + exit 2 + fi + + export STI_PRIMARY_RELEASE_TAR="${primary}" + export STI_RELEASE_COMMIT="$(cat ${STI_LOCAL_RELEASEPATH}/.commit)" +} + + # sti::build::get_version_vars loads the standard version variables as # ENV vars sti::build::get_version_vars() {