diff --git a/docs/new_labels.md b/docs/new_labels.md new file mode 100644 index 000000000..62fda6e64 --- /dev/null +++ b/docs/new_labels.md @@ -0,0 +1,27 @@ +# Adding New Labels +New Docker Labels may be created and/or updated for the output image via the image metadata file. + +If a new label is specified in the metadata file, the label will be added in the output image. However, any label previously defined in the base builder image will be ***overwritten*** in the output image, if the same label name is specified in the image metadata file. + +## Image Metadata File Name and Path +The name and path of the file ***must*** be the following: +```bash +/tmp/.s2i/image_metadata.json +``` + +## Example +The file may have one or more label/value pairs. Below is the JSON format of the labels, in the image metadata file: +```bash +{ + "labels": [ + {"labelkey1":"value1"}, + {"labelkey2":"value2"}, + ......... + ] +} + +``` +Note: If the JSON format is different than shown above, it will cause an error. + +## Creating the File +The file should be created during the `assemble` step. diff --git a/pkg/build/strategies/sti/postexecutorstep.go b/pkg/build/strategies/sti/postexecutorstep.go index a596ad23c..95821367e 100644 --- a/pkg/build/strategies/sti/postexecutorstep.go +++ b/pkg/build/strategies/sti/postexecutorstep.go @@ -2,6 +2,7 @@ package sti import ( "archive/tar" + "encoding/json" "fmt" "io" "io/ioutil" @@ -19,6 +20,8 @@ import ( utilstatus "github.com/openshift/source-to-image/pkg/util/status" ) +const maximumLabelSize = 10240 + type postExecutorStepContext struct { // id of the previous image that we're holding because after committing the image, we'll lose it. // Used only when build is incremental and RemovePreviousImage setting is enabled. @@ -104,6 +107,8 @@ type commitImageStep struct { image string builder *STI docker dockerpkg.Docker + fs util.FileSystem + tar s2itar.Tar } func (step *commitImageStep) execute(ctx *postExecutorStepContext) error { @@ -116,8 +121,16 @@ func (step *commitImageStep) execute(ctx *postExecutorStepContext) error { cmd := createCommandForExecutingRunScript(step.builder.scriptsURL, ctx.destination) + if err = checkAndGetNewLabels(step.builder, step.docker, step.tar, ctx.containerID); err != nil { + return fmt.Errorf("could not check for new labels for %q image: %v", step.image, err) + } + ctx.labels = createLabelsForResultingImage(step.builder, step.docker, step.image) + if err = checkLabelSize(ctx.labels); err != nil { + return fmt.Errorf("label validation failed for %q image: %v", step.image, err) + } + // Set the image entrypoint back to its original value on commit, the running // container has "env" as its entrypoint and we don't want to commit that. entrypoint, err := step.docker.GetImageEntrypoint(step.image) @@ -211,46 +224,10 @@ func (step *downloadFilesFromBuilderImageStep) execute(ctx *postExecutorStepCont } func (step *downloadFilesFromBuilderImageStep) downloadAndExtractFile(artifactPath, artifactsDir, containerID string) error { - glog.V(5).Infof("Downloading file %q", artifactPath) - - fd, err := ioutil.TempFile(artifactsDir, "s2i-runtime-artifact") - if err != nil { - step.builder.result.BuildInfo.FailureReason = utilstatus.NewFailureReason( - utilstatus.ReasonFSOperationFailed, - utilstatus.ReasonMessageFSOperationFailed, - ) - return fmt.Errorf("could not create temporary file for runtime artifact: %v", err) + if res, err := downloadAndExtractFileFromContainer(step.docker, step.tar, artifactPath, artifactsDir, containerID); err != nil { + step.builder.result.BuildInfo.FailureReason = res + return err } - defer func() { - fd.Close() - os.Remove(fd.Name()) - }() - - if err := step.docker.DownloadFromContainer(artifactPath, fd, containerID); err != nil { - step.builder.result.BuildInfo.FailureReason = utilstatus.NewFailureReason( - utilstatus.ReasonGenericS2IBuildFailed, - utilstatus.ReasonMessageGenericS2iBuildFailed, - ) - return fmt.Errorf("could not download file (%q -> %q) from container %s: %v", artifactPath, fd.Name(), containerID, err) - } - - // after writing to the file descriptor we need to rewind pointer to the beginning of the file before next reading - if _, err := fd.Seek(0, os.SEEK_SET); err != nil { - step.builder.result.BuildInfo.FailureReason = utilstatus.NewFailureReason( - utilstatus.ReasonGenericS2IBuildFailed, - utilstatus.ReasonMessageGenericS2iBuildFailed, - ) - return fmt.Errorf("could not seek to the beginning of the file %q: %v", fd.Name(), err) - } - - if err := step.tar.ExtractTarStream(artifactsDir, fd); err != nil { - step.builder.result.BuildInfo.FailureReason = utilstatus.NewFailureReason( - utilstatus.ReasonGenericS2IBuildFailed, - utilstatus.ReasonMessageGenericS2iBuildFailed, - ) - return fmt.Errorf("could not extract runtime artifact %q into the directory %q: %v", artifactPath, artifactsDir, err) - } - return nil } @@ -450,22 +427,20 @@ func createLabelsForResultingImage(builder *STI, docker dockerpkg.Docker, baseIm } configLabels := builder.config.Labels + newLabels := builder.newLabels - return mergeLabels(configLabels, generatedLabels, existingLabels) + return mergeLabels(configLabels, generatedLabels, existingLabels, newLabels) } -func mergeLabels(configLabels, generatedLabels, existingLabels map[string]string) map[string]string { - result := map[string]string{} - for k, v := range existingLabels { - result[k] = v +func mergeLabels(labels ...map[string]string) map[string]string { + mergedLabels := map[string]string{} + + for _, labelMap := range labels { + for k, v := range labelMap { + mergedLabels[k] = v + } } - for k, v := range generatedLabels { - result[k] = v - } - for k, v := range configLabels { - result[k] = v - } - return result + return mergedLabels } func createCommandForExecutingRunScript(scriptsURL map[string]string, location string) string { @@ -482,3 +457,115 @@ func createCommandForExecutingRunScript(scriptsURL map[string]string, location s } return cmd } + +func downloadAndExtractFileFromContainer(docker dockerpkg.Docker, tar s2itar.Tar, sourcePath, destinationPath, containerID string) (api.FailureReason, error) { + glog.V(5).Infof("Downloading file %q", sourcePath) + + fd, err := ioutil.TempFile(destinationPath, "s2i-runtime-artifact") + if err != nil { + res := utilstatus.NewFailureReason( + utilstatus.ReasonFSOperationFailed, + utilstatus.ReasonMessageFSOperationFailed, + ) + return res, fmt.Errorf("could not create temporary file for runtime artifact: %v", err) + } + defer func() { + fd.Close() + os.Remove(fd.Name()) + }() + + if err := docker.DownloadFromContainer(sourcePath, fd, containerID); err != nil { + res := utilstatus.NewFailureReason( + utilstatus.ReasonGenericS2IBuildFailed, + utilstatus.ReasonMessageGenericS2iBuildFailed, + ) + return res, fmt.Errorf("could not download file (%q -> %q) from container %s: %v", sourcePath, fd.Name(), containerID, err) + } + + // after writing to the file descriptor we need to rewind pointer to the beginning of the file before next reading + if _, err := fd.Seek(0, os.SEEK_SET); err != nil { + res := utilstatus.NewFailureReason( + utilstatus.ReasonGenericS2IBuildFailed, + utilstatus.ReasonMessageGenericS2iBuildFailed, + ) + return res, fmt.Errorf("could not seek to the beginning of the file %q: %v", fd.Name(), err) + } + + if err := tar.ExtractTarStream(destinationPath, fd); err != nil { + res := utilstatus.NewFailureReason( + utilstatus.ReasonGenericS2IBuildFailed, + utilstatus.ReasonMessageGenericS2iBuildFailed, + ) + return res, fmt.Errorf("could not extract artifact %q into the directory %q: %v", sourcePath, destinationPath, err) + } + + return utilstatus.NewFailureReason("", ""), nil +} + +func checkLabelSize(labels map[string]string) error { + var sum = 0 + for k, v := range labels { + sum += len(k) + len(v) + } + + if sum > maximumLabelSize { + return fmt.Errorf("label size '%d' exceeds the maximum limit '%d'", sum, maximumLabelSize) + } + + return nil +} + +// check for new labels and apply to the output image. +func checkAndGetNewLabels(builder *STI, docker dockerpkg.Docker, tar s2itar.Tar, containerID string) error { + glog.V(3).Infof("Checking for new Labels to apply... ") + + // metadata filename and its path inside the container + metadataFilename := "image_metadata.json" + sourceFilepath := filepath.Join("/tmp/.s2i", metadataFilename) + + // create the 'downloadPath' folder if it doesn't exist + downloadPath := filepath.Join(builder.config.WorkingDir, "metadata") + glog.V(3).Infof("Creating the download path '%s'", downloadPath) + if err := os.MkdirAll(downloadPath, 0700); err != nil { + glog.Errorf("Error creating dir %q for '%s': %v", downloadPath, metadataFilename, err) + return err + } + + // download & extract the file from container + if _, err := downloadAndExtractFileFromContainer(docker, tar, sourceFilepath, downloadPath, containerID); err != nil { + glog.V(3).Infof("unable to download and extract '%s' ... continuing", metadataFilename) + return nil + } + + // open the file + filePath := filepath.Join(downloadPath, metadataFilename) + fd, err := os.Open(filePath) + if fd == nil || err != nil { + return fmt.Errorf("unable to open file '%s' : %v", downloadPath, err) + } + defer fd.Close() + + // read the file to a string + str, err := ioutil.ReadAll(fd) + if err != nil { + return fmt.Errorf("error reading file '%s' in to a string: %v", filePath, err) + } + glog.V(3).Infof("new Labels File contents : \n%s\n", str) + + // string into a map + var data map[string]interface{} + + if err = json.Unmarshal([]byte(str), &data); err != nil { + return fmt.Errorf("JSON Unmarshal Error with '%s' file : %v", metadataFilename, err) + } + + // update newLabels[] + labels := data["labels"] + for _, l := range labels.([]interface{}) { + for k, v := range l.(map[string]interface{}) { + builder.newLabels[k] = v.(string) + } + } + + return nil +} diff --git a/pkg/build/strategies/sti/sti.go b/pkg/build/strategies/sti/sti.go index fcf77e3ad..ed85e352a 100644 --- a/pkg/build/strategies/sti/sti.go +++ b/pkg/build/strategies/sti/sti.go @@ -65,6 +65,7 @@ type STI struct { incremental bool sourceInfo *api.SourceInfo env []string + newLabels map[string]string // Interfaces preparer build.Preparer @@ -124,6 +125,7 @@ func New(client dockerpkg.Client, config *api.Config, fs util.FileSystem, overri externalScripts: map[string]bool{}, installedScripts: map[string]bool{}, scriptsURL: map[string]string{}, + newLabels: map[string]string{}, } if len(config.RuntimeImage) > 0 { @@ -707,6 +709,8 @@ func (builder *STI) initPostExecutorSteps() { image: builder.config.BuilderImage, builder: builder, docker: builder.docker, + fs: builder.fs, + tar: builder.tar, }, &reportSuccessStep{ builder: builder, diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 9fd45b1bb..790b9fba4 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -23,6 +23,7 @@ import ( "github.com/openshift/source-to-image/pkg/api" "github.com/openshift/source-to-image/pkg/build/strategies" "github.com/openshift/source-to-image/pkg/docker" + dockerpkg "github.com/openshift/source-to-image/pkg/docker" "github.com/openshift/source-to-image/pkg/tar" "github.com/openshift/source-to-image/pkg/util" "golang.org/x/net/context" @@ -188,48 +189,53 @@ func integration(t *testing.T) *integrationTest { // Test a clean build. The simplest case. func TestCleanBuild(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, "", true, true) + integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, "", true, true, false) +} + +// Test Labels +func TestCleanBuildLabel(t *testing.T) { + integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, "", true, true, true) } func TestCleanBuildUser(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuildUser, false, FakeUserImage, "", true, true) + integration(t).exerciseCleanBuild(TagCleanBuildUser, false, FakeUserImage, "", true, true, false) } func TestCleanBuildFileScriptsURL(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, FakeScriptsFileURL, true, true) + integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, FakeScriptsFileURL, true, true, false) } func TestCleanBuildHttpScriptsURL(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, FakeScriptsHTTPURL, true, true) + integration(t).exerciseCleanBuild(TagCleanBuild, false, FakeBuilderImage, FakeScriptsHTTPURL, true, true, false) } func TestCleanBuildScripts(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuildScripts, false, FakeImageScripts, "", true, true) + integration(t).exerciseCleanBuild(TagCleanBuildScripts, false, FakeImageScripts, "", true, true, false) } func TestLayeredBuildNoTar(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanLayeredBuildNoTar, false, FakeImageNoTar, FakeScriptsFileURL, false, true) + integration(t).exerciseCleanBuild(TagCleanLayeredBuildNoTar, false, FakeImageNoTar, FakeScriptsFileURL, false, true, false) } // Test that a build config with a callbackURL will invoke HTTP endpoint func TestCleanBuildCallbackInvoked(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuild, true, FakeBuilderImage, "", true, true) + integration(t).exerciseCleanBuild(TagCleanBuild, true, FakeBuilderImage, "", true, true, false) } func TestCleanBuildOnBuild(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuildOnBuild, false, FakeImageOnBuild, "", true, true) + integration(t).exerciseCleanBuild(TagCleanBuildOnBuild, false, FakeImageOnBuild, "", true, true, false) } func TestCleanBuildOnBuildNoName(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuildOnBuildNoName, false, FakeImageOnBuild, "", false, false) + integration(t).exerciseCleanBuild(TagCleanBuildOnBuildNoName, false, FakeImageOnBuild, "", false, false, false) } func TestCleanBuildNoName(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanBuildNoName, false, FakeBuilderImage, "", true, false) + integration(t).exerciseCleanBuild(TagCleanBuildNoName, false, FakeBuilderImage, "", true, false, false) } func TestLayeredBuildNoTarNoName(t *testing.T) { - integration(t).exerciseCleanBuild(TagCleanLayeredBuildNoTarNoName, false, FakeImageNoTar, FakeScriptsFileURL, false, false) + integration(t).exerciseCleanBuild(TagCleanLayeredBuildNoTarNoName, false, FakeImageNoTar, FakeScriptsFileURL, false, false, false) } func TestAllowedUIDsNamedUser(t *testing.T) { @@ -270,7 +276,7 @@ func (i *integrationTest) exerciseCleanAllowedUIDsBuild(tag, imageName string, e } } -func (i *integrationTest) exerciseCleanBuild(tag string, verifyCallback bool, imageName string, scriptsURL string, expectImageName bool, setTag bool) { +func (i *integrationTest) exerciseCleanBuild(tag string, verifyCallback bool, imageName string, scriptsURL string, expectImageName bool, setTag bool, checkLabel bool) { t := i.t callbackURL := "" callbackInvoked := false @@ -343,6 +349,11 @@ func (i *integrationTest) exerciseCleanBuild(tag string, verifyCallback bool, im i.checkForImage(tag) containerID := i.createContainer(tag) i.checkBasicBuildState(containerID, resp.WorkingDir) + + if checkLabel { + i.checkForLabel(tag) + } + i.removeContainer(containerID) } @@ -640,3 +651,16 @@ func (i *integrationTest) fileExistsInContainer(cID string, filePath string) boo defer rdr.Close() return "" != stats.Name } + +func (i *integrationTest) checkForLabel(image string) { + docker := dockerpkg.New(engineClient, (&api.Config{}).PullAuthentication) + + labelMap, err := docker.GetLabels(image) + if err != nil { + i.t.Fatalf("Unable to get labels from image %s: %v", image, err) + } + + if labelMap["testLabel"] != "testLabel_value" { + i.t.Errorf("Unable to verify 'testLabel' for image '%s'", image) + } +} diff --git a/test/integration/scripts/.s2i/bin/assemble b/test/integration/scripts/.s2i/bin/assemble index ed19662f4..c07ddb51a 100755 --- a/test/integration/scripts/.s2i/bin/assemble +++ b/test/integration/scripts/.s2i/bin/assemble @@ -14,3 +14,12 @@ fi if [ -e /tmp/artifacts/save-artifacts-invoked ]; then touch /sti-fake/save-artifacts-invoked fi + +mkdir -p /tmp/.s2i/ +cat > /tmp/.s2i/image_metadata.json << EOL +{ + "labels": [ + {"testLabel": "testLabel_value"} + ] +} +EOL