commit 0cfa25360682f66069d595fb0ede0fcc69bad41f Author: Joshua Dolitsky Date: Tue Sep 19 01:02:12 2017 -0500 the museum is now open diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..87537aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM alpine:3.6 +RUN apk add --no-cache ca-certificates +COPY bin/linux/amd64/chartmuseum /chartmuseum +ENTRYPOINT ["/chartmuseum"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6f256bf --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 ChartMuseum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..130187c --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +# Change this and commit to create new release +VERSION=0.1.0 +REVISION := $(shell git rev-parse --short HEAD;) + +HAS_GLIDE := $(shell command -v glide;) +HAS_PIP := $(shell command -v pip;) +HAS_VENV := $(shell command -v virtualenv;) +HAS_GOVIZ := $(shell command -v goviz;) +HAS_DOT := $(shell command -v dot;) +HAS_AWS := $(shell command -v aws;) + +.PHONY: bootstrap +bootstrap: +ifndef HAS_GLIDE + @go get -u github.com/Masterminds/glide +endif + @glide install --strip-vendor + +.PHONY: build +build: export GOARCH=amd64 +build: export CGO_ENABLED=0 +build: + @GOOS=linux go build -v -i --ldflags="-w -X main.Version=$(VERSION) -X main.Revision=$(REVISION)" \ + -o bin/linux/amd64/chartmuseum cmd/chartmuseum/main.go # linux + @GOOS=darwin go build -v -i --ldflags="-w -X main.Version=$(VERSION) -X main.Revision=$(REVISION)" \ + -o bin/darwin/amd64/chartmuseum cmd/chartmuseum/main.go # mac osx + +.PHONY: clean +clean: + @git status --ignored --short | grep '^!! ' | sed 's/!! //' | xargs rm -rf + +.PHONY: setup-test-environment +setup-test-environment: +ifndef HAS_PIP + @sudo apt-get update && sudo apt-get install -y python-pip +endif +ifndef HAS_VENV + @sudo pip install virtualenv +endif + @./scripts/setup_test_environment.sh + +.PHONY: test +test: setup-test-environment + @./scripts/test.sh + +.PHONY: testcloud +testcloud: export TEST_CLOUD_STORAGE=1 +testcloud: test + +.PHONY: covhtml +covhtml: + @go tool cover -html=.cover/cover.out + +.PHONY: acceptance +acceptance: setup-test-environment + @./scripts/acceptance.sh + +.PHONY: run +run: + @rm -rf .chartstorage/ + @bin/darwin/amd64/chartmuseum --debug --port=8080 --storage="local" \ + --storage-local-rootdir=".chartstorage/" + +.PHONY: tree +tree: + @tree -I vendor + +# https://github.com/hirokidaichi/goviz/pull/8 +.PHONY: goviz +goviz: +ifndef HAS_GOVIZ + @go get -u github.com/RobotsAndPencils/goviz +endif +ifndef HAS_DOT + @sudo apt-get update && sudo apt-get install -y graphviz +endif + @goviz -i github.com/chartmuseum/chartmuseum/cmd/chartmuseum -l | dot -Tpng -o goviz.png + +.PHONY: release +release: +ifndef HAS_AWS + @sudo pip install awscli +endif + @scripts/release.sh $(VERSION) \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ffbeaa6 --- /dev/null +++ b/README.md @@ -0,0 +1,150 @@ +# ChartMuseum + + +[![CircleCI](https://circleci.com/gh/chartmuseum/chartmuseum.svg?style=svg)](https://circleci.com/gh/chartmuseum/chartmuseum) +[![Go Report Card](https://goreportcard.com/badge/github.com/chartmuseum/chartmuseum)](https://goreportcard.com/report/github.com/chartmuseum/chartmuseum) +[![GoDoc](https://godoc.org/github.com/chartmuseum/chartmuseum?status.svg)](https://godoc.org/github.com/chartmuseum/chartmuseum) +**_"Preserve your precious artifacts... in the cloud!"_** + +*ChartMuseum* is an open-source **[Helm Chart Repository](https://github.com/kubernetes/helm/blob/master/docs/chart_repository.md)** written in Go (Golang), with support for cloud storage backends, including [Google Cloud Storage](https://cloud.google.com/storage/) and [Amazon S3](https://aws.amazon.com/s3/). + +Works as a valid Helm Chart Repository, and also provides an API for uploading new chart packages to storage etc. + + + + +Powered by some great Go technology: +- [Kubernetes Helm](https://github.com/kubernetes/helm) - for working with charts, generating repository index +- [Gin Web Framework](https://github.com/gin-gonic/gin) - for HTTP routing +- [cli](https://github.com/urfave/cli) - for command line option parsing +- [zap](https://github.com/uber-go/zap) - for logging + +## API +### Helm Chart Repository +- `GET /index.yaml` - retrieved when you run `helm repo add chartmuseum http://localhost:8080/` +- `GET /charts/mychart-0.1.0.tgz` - retrieved when you run `helm install chartmuseum/mychart` +- `GET /charts/mychart-0.1.0.tgz.prov` - retrieved when you run `helm install` with the `--verify` flag + +### Chart Manipulation +- `POST /api/charts` - upload a new chart version +- `POST /api/prov` - upload a new provenance file +- `DELETE /api/charts//` - delete a chart version (and corresponding provenance file) +- `GET /api/charts` - list all charts +- `GET /api/charts/` - list all versions of a chart +- `GET /api/charts//` - describe a chart version + +## Uploading a Chart Package +*Follow **"How to Run"** section below to get ChartMuseum up and running at http://localhost:8080* + +First create `mychart-0.1.0.tgz` using the [Helm CLI](https://docs.helm.sh/using_helm/#installing-helm): +``` +cd mychart/ +helm package . +``` + +Upload `mychart-0.1.0.tgz`: +```bash +curl --data-binary "@mychart-0.1.0.tgz" http://localhost:8080/api/charts +``` + +If you've signed your package and generated a [provenance file](https://github.com/kubernetes/helm/blob/master/docs/provenance.md), upload it with: +```bash +curl --data-binary "@mychart-0.1.0.tgz.prov" http://localhost:8080/api/prov +``` + +## Installing Charts into Kubernetes +Add the URL to your *ChartMuseum* installation to the local repository list: +```bash +helm repo add chartmuseum http://localhost:8080 +``` + +Search for charts: +```bash +helm search chartmuseum/ +``` + +Install chart: +```bash +helm install chartmuseum/mychart +``` + +## How to Run +### CLI +#### Installation +Install the binary: +```bash +# on Linux +curl -LO https://s3.amazonaws.com/chartmuseum/release/latest/bin/linux/amd64/chartmuseum + +# on macOS +curl -LO https://s3.amazonaws.com/chartmuseum/release/latest/bin/darwin/amd64/chartmuseum + +chmod +x ./chartmuseum +mv ./chartmuseum /usr/local/bin +``` +Using `latest` in URLs above will get the latest binary (built from master branch). + +Replace `latest` with `$(curl -s https://s3.amazonaws.com/chartmuseum/release/stable.txt)` to automatically determine the latest stable release (e.g. `v0.1.0`). + +Show all CLI options with `chartmuseum --help` and determine version with `chartmuseum --version` + +#### Using with Amazon S3 +Make sure your environment is properly setup to access `my-s3-bucket` +```bash +chartmuseum --debug --port=8080 \ + --storage="amazon" \ + --storage-amazon-bucket="my-s3-bucket" \ + --storage-amazon-prefix="" \ + --storage-amazon-region="us-east-1" +``` + +#### Using with Google Cloud Storage +Make sure your environment is properly setup to access `my-gcs-bucket` +```bash +chartmuseum --debug --port=8080 \ + --storage="google" \ + --storage-google-bucket="my-gcs-bucket" \ + --storage-google-prefix="" +``` + +#### Using with local filesystem storage +Make sure you have read-write access to `./chartstorage` (will create if doesn't exist) +```bash +chartmuseum --debug --port=8080 \ + --storage="local" \ + --storage-local-rootdir="./chartstorage" +``` + +### Docker Image +Available via [Docker Hub](https://hub.docker.com/r/chartmuseum/chartmuseum/). + +Example usage (S3): +```bash +docker run --rm -it \ + -p 8080:8080 \ + -v ~/.aws:/root/.aws:ro \ + chartmuseum/chartmuseum:latest \ + --debug --port=8080 \ + --storage="amazon" \ + --storage-amazon-bucket="my-s3-bucket" \ + --storage-amazon-prefix="" \ + --storage-amazon-region="us-east-1" +``` + +## Notes on index.yaml +The repository index (index.yaml) is dynamically generated based on packages found in storage. If you store your own version of index.yaml, it will be completely ignored. + +`GET /index.yaml` occurs when you run `helm repo add chartmuseum http://localhost:8080/` or `helm repo update`. + +If you manually add/remove a .tgz package from storage, it will be immediately reflected in `GET /index.yaml`. + +You are no longer required to maintain your own version of index.yaml using `helm repo index --merge`. + +## Mirroring the official Kubernetes repositories +Please see `scripts/mirror_k8s_repos.sh` for an example of how to download all .tgz packages from the official Kubernetes repositories (both stable and incubator). + +You can then use *ChartMuseum* to serve up an internal mirror: +``` +scripts/mirror_k8s_repos.sh +chartmuseum --debug --port=8080 --storage="local" --storage-local-rootdir="./mirror" + ``` diff --git a/acceptance_tests/helm.robot b/acceptance_tests/helm.robot new file mode 100644 index 0000000..14c53ac --- /dev/null +++ b/acceptance_tests/helm.robot @@ -0,0 +1,83 @@ +*** Settings *** +Documentation Tests to verify that ChartMuseum is able to work with +... Helm CLI and act as a valid Helm Chart Repository using +... all supported storage backends (local, s3, gcs). +Library OperatingSystem +Library lib/ChartMuseum.py +Library lib/Helm.py +Suite Setup Suite Setup +Suite Teardown Suite Teardown + +*** Test Cases *** +ChartMuseum works with Helm using local storage + Test Helm integration local + +ChartMuseum works with Helm using Amazon cloud storage + Test Helm integration amazon + +ChartMuseum works with Helm using Google cloud storage + Test Helm integration google + +*** Keyword *** +Test Helm integration + [Arguments] ${storage} + Start ChartMuseum server with storage backend ${storage} + Able to add ChartMuseum as Helm chart repo + Helm search does not return test charts + Unable to fetch and verify test charts + Upload test charts to ChartMuseum + Upload provenance files to ChartMuseum + Able to update ChartMuseum repo + Helm search returns test charts + Able to fetch and verify test charts + Delete test charts from ChartMuseum + Able to update ChartMuseum repo + Helm search does not return test charts + Unable to fetch and verify test charts + +Start ChartMuseum server with storage backend + [Arguments] ${storage} + ChartMuseum.start chartmuseum ${storage} + Sleep 2 + +Upload test charts to ChartMuseum + ChartMuseum.upload test charts + +Upload provenance files to ChartMuseum + ChartMuseum.upload provenance files + +Delete test charts from ChartMuseum + ChartMuseum.delete test charts + +Able to add ChartMuseum as Helm chart repo + Helm.add chart repo + Helm.return code should be 0 + Helm.output contains has been added + +Able to update ChartMuseum repo + Helm.update chart repos + Helm.return code should be 0 + +Helm search returns test charts + Helm.search for chart mychart + Helm.output contains mychart + +Helm search does not return test charts + Helm.search for chart mychart + Helm.output does not contain mychart + +Able to fetch and verify test charts + Helm.fetch and verify chart mychart + Helm.return code should be 0 + +Unable to fetch and verify test charts + Helm.fetch and verify chart mychart + Helm.return code should not be 0 + +Suite Setup + ChartMuseum.remove chartmuseum logs + +Suite Teardown + Helm.remove chart repo + ChartMuseum.stop chartmuseum + ChartMuseum.print chartmuseum logs \ No newline at end of file diff --git a/acceptance_tests/lib/ChartMuseum.py b/acceptance_tests/lib/ChartMuseum.py new file mode 100644 index 0000000..500f6fc --- /dev/null +++ b/acceptance_tests/lib/ChartMuseum.py @@ -0,0 +1,98 @@ +import glob +import os +import requests +import shutil + +import common + + +class ChartMuseum(common.CommandRunner): + def http_status_code_should_be(self, expected_status, actual_status): + if int(expected_status) != int(actual_status): + raise AssertionError('Expected HTTP status code to be %s but was %s.' + % (expected_status, actual_status)) + + def start_chartmuseum(self, storage): + self.stop_chartmuseum() + os.chdir(self.rootdir) + cmd = 'chartmuseum --debug --port=%d --storage="%s" ' % (common.PORT, storage) + if storage == 'local': + shutil.rmtree(common.STORAGE_DIR, ignore_errors=True) + cmd += '--storage-local-rootdir=%s >> %s 2>&1' % (common.STORAGE_DIR, common.LOGFILE) + elif storage == 'amazon': + cmd += '--storage-amazon-bucket="%s" --storage-amazon-prefix="%s" --storage-amazon-region="%s" >> %s 2>&1' \ + % (common.STORAGE_AMAZON_BUCKET, common.STORAGE_AMAZON_PREFIX, common.STORAGE_AMAZON_REGION, common.LOGFILE) + elif storage == 'google': + cmd += '--storage-google-bucket="%s" --storage-google-prefix="%s" >> %s 2>&1' \ + % (common.STORAGE_GOOGLE_BUCKET, common.STORAGE_GOOGLE_PREFIX, common.LOGFILE) + print(cmd) + self.run_command(cmd, detach=True) + + def stop_chartmuseum(self): + self.run_command('pkill -9 chartmuseum') + shutil.rmtree(common.STORAGE_DIR, ignore_errors=True) + + def remove_chartmuseum_logs(self): + os.chdir(self.rootdir) + self.run_command('rm -f %s' % common.LOGFILE) + + def print_chartmuseum_logs(self): + os.chdir(self.rootdir) + self.run_command('cat %s' % common.LOGFILE) + + def upload_test_charts(self): + charts_endpoint = '%s/api/charts' % common.HELM_REPO_URL + testcharts_dir = os.path.join(self.rootdir, common.TESTCHARTS_DIR) + os.chdir(testcharts_dir) + for d in os.listdir('.'): + if not os.path.isdir(d): + continue + os.chdir(d) + tgz = glob.glob('*.tgz')[0] + print('Uploading test chart package "%s"' % tgz) + with open(tgz) as f: + response = requests.post(url=charts_endpoint, data=f.read()) + print('POST %s' % charts_endpoint) + print('HTTP STATUS: %s' % response.status_code) + print('HTTP CONTENT: %s' % response.content) + self.http_status_code_should_be(201, response.status_code) + os.chdir('../') + + def upload_provenance_files(self): + prov_endpoint = '%s/api/prov' % common.HELM_REPO_URL + testcharts_dir = os.path.join(self.rootdir, common.TESTCHARTS_DIR) + os.chdir(testcharts_dir) + for d in os.listdir('.'): + if not os.path.isdir(d): + continue + os.chdir(d) + prov = glob.glob('*.tgz.prov')[0] + print('Uploading provenance file "%s"' % prov) + with open(prov) as f: + response = requests.post(url=prov_endpoint, data=f.read()) + print('POST %s' % prov_endpoint) + print('HTTP STATUS: %s' % response.status_code) + print('HTTP CONTENT: %s' % response.content) + self.http_status_code_should_be(201, response.status_code) + os.chdir('../') + + def delete_test_charts(self): + endpoint = '%s/api/charts' % common.HELM_REPO_URL + testcharts_dir = os.path.join(self.rootdir, common.TESTCHARTS_DIR) + os.chdir(testcharts_dir) + for d in os.listdir('.'): + if not os.path.isdir(d): + continue + os.chdir(d) + tgz = glob.glob('*.tgz')[0] + tmp = tgz[:-4].rsplit('-', 1) + name = tmp[0] + version = tmp[1] + print('Delete test chart "%s-%s"' % (name, version)) + with open(tgz) as f: + epoint = '%s/%s/%s' % (endpoint, name, version) + response = requests.delete(url=epoint) + print('HTTP STATUS: %s' % response.status_code) + print('HTTP CONTENT: %s' % response.content) + self.http_status_code_should_be(200, response.status_code) + os.chdir('../') \ No newline at end of file diff --git a/acceptance_tests/lib/Helm.py b/acceptance_tests/lib/Helm.py new file mode 100644 index 0000000..05ad3b6 --- /dev/null +++ b/acceptance_tests/lib/Helm.py @@ -0,0 +1,26 @@ +import os + +import common + + +class Helm(common.CommandRunner): + def add_chart_repo(self): + self.remove_chart_repo() + self.run_command('helm repo add %s %s' % (common.HELM_REPO_NAME, common.HELM_REPO_URL)) + + def remove_chart_repo(self): + self.run_command('helm repo remove %s' % common.HELM_REPO_NAME) + + def search_for_chart(self, chart): + self.run_command('helm search %s/%s' % (common.HELM_REPO_NAME, chart)) + + def update_chart_repos(self): + # "| head -n -1" prevents UnicodeDecodeError due to last line of output + self.run_command('helm repo update | head -n -1 | \ + grep "Successfully got an update from the \\"%s\\""' \ + % common.HELM_REPO_NAME) + + def fetch_and_verify_chart(self, chart): + os.chdir(self.rootdir) + os.chdir(common.ACCEPTANCE_DIR) + self.run_command('helm fetch --verify --keyring ../%s %s/%s' % (common.KEYRING, common.HELM_REPO_NAME, chart)) diff --git a/acceptance_tests/lib/__init__.py b/acceptance_tests/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/acceptance_tests/lib/common.py b/acceptance_tests/lib/common.py new file mode 100644 index 0000000..58ce938 --- /dev/null +++ b/acceptance_tests/lib/common.py @@ -0,0 +1,56 @@ +import os +import subprocess +import time + +NOW = time.strftime('%Y%m%d%H%M%S') +PORT = 8080 +HELM_REPO_NAME = 'chartmuseum' +HELM_REPO_URL = 'http://localhost:%d' % PORT +TESTCHARTS_DIR = 'testdata/charts' +ACCEPTANCE_DIR = '.acceptance/' +STORAGE_DIR = os.path.join(ACCEPTANCE_DIR, 'storage/') +KEYRING = 'testdata/pgp/helm-test-key.pub' +LOGFILE = '.chartmuseum.log' + +STORAGE_AMAZON_BUCKET = os.environ['TEST_STORAGE_AMAZON_BUCKET'] +STORAGE_AMAZON_REGION = os.environ['TEST_STORAGE_AMAZON_REGION'] +STORAGE_GOOGLE_BUCKET = os.environ['TEST_STORAGE_GOOGLE_BUCKET'] + +STORAGE_AMAZON_PREFIX = 'acceptance/%s' % NOW +STORAGE_GOOGLE_PREFIX = 'acceptance/%s' % NOW + + +class CommandRunner(object): + def __init__(self): + self.rc = 0 + self.pid = 0 + self.stdout = '' + self.rootdir = os.path.realpath(os.path.join(__file__, '../../../')) + + def return_code_should_be(self, expected_rc): + if int(expected_rc) != self.rc: + raise AssertionError('Expected return code to be "%s" but was "%s".' + % (expected_rc, self.rc)) + + def return_code_should_not_be(self, expected_rc): + if int(expected_rc) == self.rc: + raise AssertionError('Expected return code not to be "%s".' % expected_rc) + + def output_contains(self, s): + if s not in self.stdout: + raise AssertionError('Output does not contain "%s".' % s) + + def output_does_not_contain(self, s): + if s in self.stdout: + raise AssertionError('Output contains "%s".' % s) + + def run_command(self, command, detach=False): + process = subprocess.Popen(['/bin/bash', '-xc', command], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + if not detach: + stdout = process.communicate()[0].strip() + print(stdout) + self.rc = process.returncode + # Remove debug lines that start with "+ " + self.stdout = '\n'.join(filter(lambda x: not x.startswith('+ '), stdout.split('\n'))) diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..f78dcbf --- /dev/null +++ b/circle.yml @@ -0,0 +1,46 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: + docker: + - image: circleci/golang:1.8 + environment: + GOOGLE_APPLICATION_CREDENTIALS: /home/circleci/gcp-key.json + working_directory: /go/src/github.com/chartmuseum/chartmuseum + steps: + # Setup build environment + - checkout + - setup_remote_docker + - run: sudo mkdir /usr/local/go/pkg/darwin_amd64 + - run: sudo chown -R circleci:circleci /usr/local/go/pkg/darwin_amd64 /usr/local/go/pkg/linux_amd64 + - run: echo $GCLOUD_SERVICE_KEY | base64 --decode --ignore-garbage > ${HOME}/gcp-key.json + - run: mkdir -p ${HOME}/.docker + - run: echo $DOCKERHUB_CONFIG | base64 --decode --ignore-garbage > ${HOME}/.docker/config.json + + # Build steps + - run: make bootstrap + - run: make testcloud + - run: make goviz + - run: make build + - run: make acceptance + - run: if [ $CIRCLE_BRANCH == "master" ]; then make release; fi + + # Archive artifacts + - store_artifacts: + when: always + path: .cover/ + destination: .cover/ + - store_artifacts: + when: always + path: goviz.png + destination: goviz.png + - store_artifacts: + when: always + path: bin/ + destination: bin/ + - store_artifacts: + when: always + path: .robot/ + destination: .robot/ \ No newline at end of file diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go new file mode 100644 index 0000000..c91a3a9 --- /dev/null +++ b/cmd/chartmuseum/main.go @@ -0,0 +1,163 @@ +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/chartmuseum/chartmuseum/pkg/chartmuseum" + "github.com/chartmuseum/chartmuseum/pkg/storage" + + "github.com/urfave/cli" +) + +var ( + crash = log.Fatal + newServer = chartmuseum.NewServer + + // Version is the semantic version (added at compile time) + Version string + + // Revision is the git commit id (added at compile time) + Revision string +) + +func main() { + app := cli.NewApp() + app.Name = "ChartMuseum" + app.Version = fmt.Sprintf("%s (build %s)", Version, Revision) + app.Usage = "Helm Chart Repository with support for Amazon S3 and Google Cloud Storage" + app.Action = cliHandler + app.Flags = cliFlags + app.Run(os.Args) +} + +func cliHandler(c *cli.Context) { + backend := backendFromContext(c) + + options := chartmuseum.ServerOptions{ + Debug: c.Bool("debug"), + LogJSON: c.Bool("log-json"), + StorageBackend: backend, + } + + server, err := newServer(options) + if err != nil { + crash(err) + } + + server.Listen(c.Int("port")) +} + +func backendFromContext(c *cli.Context) storage.Backend { + crashIfContextMissingFlags(c, []string{"storage"}) + + var backend storage.Backend + + storageFlag := strings.ToLower(c.String("storage")) + switch storageFlag { + case "local": + backend = localBackendFromContext(c) + case "amazon": + backend = amazonBackendFromContext(c) + case "google": + backend = googleBackendFromContext(c) + default: + crash("Unsupported storage backend: ", storageFlag) + } + + return backend +} + +func localBackendFromContext(c *cli.Context) storage.Backend { + crashIfContextMissingFlags(c, []string{"storage-local-rootdir"}) + return storage.Backend(storage.NewLocalFilesystemBackend( + c.String("storage-local-rootdir"), + )) +} + +func amazonBackendFromContext(c *cli.Context) storage.Backend { + crashIfContextMissingFlags(c, []string{"storage-amazon-bucket", "storage-amazon-region"}) + return storage.Backend(storage.NewAmazonS3Backend( + c.String("storage-amazon-bucket"), + c.String("storage-amazon-prefix"), + c.String("storage-amazon-region"), + )) +} + +func googleBackendFromContext(c *cli.Context) storage.Backend { + crashIfContextMissingFlags(c, []string{"storage-google-bucket"}) + return storage.Backend(storage.NewGoogleCSBackend( + c.String("storage-google-bucket"), + c.String("storage-google-prefix"), + )) +} + +func crashIfContextMissingFlags(c *cli.Context, flags []string) { + missing := []string{} + for _, flag := range flags { + if c.String(flag) == "" { + missing = append(missing, fmt.Sprintf("--%s", flag)) + } + } + if len(missing) > 0 { + crash("Missing required flags(s): ", strings.Join(missing, ", ")) + } +} + +var cliFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + Usage: "show debug messages", + EnvVar: "DEBUG", + }, + cli.BoolFlag{ + Name: "log-json", + Usage: "output structured logs as json", + EnvVar: "LOG_JSON", + }, + cli.IntFlag{ + Name: "port", + Value: 8080, + Usage: "port to listen on", + EnvVar: "PORT", + }, + cli.StringFlag{ + Name: "storage", + Usage: "storage backend, can be one of: local, amazon, google", + EnvVar: "STORAGE", + }, + cli.StringFlag{ + Name: "storage-local-rootdir", + Usage: "directory to store charts for local storage backend", + EnvVar: "STORAGE_LOCAL_ROOTDIR", + }, + cli.StringFlag{ + Name: "storage-amazon-bucket", + Usage: "s3 bucket to store charts for amazon storage backend", + EnvVar: "STORAGE_AMAZON_BUCKET", + }, + cli.StringFlag{ + Name: "storage-amazon-prefix", + Value: "", + Usage: "prefix to store charts for --storage-amazon-bucket", + EnvVar: "STORAGE_AMAZON_PREFIX", + }, + cli.StringFlag{ + Name: "storage-amazon-region", + Usage: "region of --storage-amazon-bucket", + EnvVar: "STORAGE_AMAZON_REGION", + }, + cli.StringFlag{ + Name: "storage-google-bucket", + Usage: "gcs bucket to store charts for google storage backend", + EnvVar: "STORAGE_GOOGLE_BUCKET", + }, + cli.StringFlag{ + Name: "storage-google-prefix", + Value: "", + Usage: "prefix to store charts for --storage-google-bucket", + EnvVar: "STORAGE_GOOGLE_PREFIX", + }, +} diff --git a/cmd/chartmuseum/main_test.go b/cmd/chartmuseum/main_test.go new file mode 100644 index 0000000..4fda337 --- /dev/null +++ b/cmd/chartmuseum/main_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "errors" + "fmt" + "os" + "testing" + + "github.com/chartmuseum/chartmuseum/pkg/chartmuseum" + + "github.com/stretchr/testify/suite" +) + +type MainTestSuite struct { + suite.Suite + LastCrashMessage string +} + +func (suite *MainTestSuite) SetupSuite() { + crash = func(v ...interface{}) { + suite.LastCrashMessage = fmt.Sprint(v...) + panic(v) + } + newServer = func(options chartmuseum.ServerOptions) (*chartmuseum.Server, error) { + return &chartmuseum.Server{}, errors.New("graceful crash") + } +} + +func (suite *MainTestSuite) TestMain() { + os.Args = []string{"chartmuseum"} + suite.Panics(main, "no storage") + suite.Equal("Missing required flags(s): --storage", suite.LastCrashMessage, "crashes with no storage") + + os.Args = []string{"chartmuseum", "--storage", "garage"} + suite.Panics(main, "bad storage") + suite.Equal("Unsupported storage backend: garage", suite.LastCrashMessage, "crashes with bad storage") + + os.Args = []string{"chartmuseum", "--storage", "local", "--storage-local-rootdir", "../../.chartstorage"} + suite.Panics(main, "local storage") + suite.Equal("graceful crash", suite.LastCrashMessage, "no error with local backend") + + os.Args = []string{"chartmuseum", "--storage", "amazon", "--storage-amazon-bucket", "x", "--storage-amazon-region", "x"} + suite.Panics(main, "amazon storage") + suite.Equal("graceful crash", suite.LastCrashMessage, "no error with amazon backend") + + os.Args = []string{"chartmuseum", "--storage", "google", "--storage-google-bucket", "x"} + suite.Panics(main, "google storage") + suite.Equal("graceful crash", suite.LastCrashMessage, "no error with google backend") +} + +func TestMainTestSuite(t *testing.T) { + suite.Run(t, new(MainTestSuite)) +} diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..ffb7446 --- /dev/null +++ b/glide.lock @@ -0,0 +1,231 @@ +hash: 00a8ed3ef68ac39e1e33ce80b62a682c17ce8e89df46e99fac16a25b9be2b413 +updated: 2017-09-18T21:57:40.784833011-05:00 +imports: +- name: cloud.google.com/go + version: 0f0b8420cb699ac4ce059c63bac263f4301fe95b + subpackages: + - compute/metadata + - iam + - internal + - internal/optional + - internal/version + - storage +- name: github.com/aws/aws-sdk-go + version: 5e436e55ac5eddc739f26a2a209b3f4248ee8e0e + subpackages: + - aws + - aws/awserr + - aws/awsutil + - aws/client + - aws/client/metadata + - aws/corehandlers + - aws/credentials + - aws/credentials/ec2rolecreds + - aws/credentials/endpointcreds + - aws/credentials/stscreds + - aws/defaults + - aws/ec2metadata + - aws/endpoints + - aws/request + - aws/session + - aws/signer/v4 + - internal/shareddefaults + - private/protocol + - private/protocol/query + - private/protocol/query/queryutil + - private/protocol/rest + - private/protocol/restxml + - private/protocol/xml/xmlutil + - service/s3 + - service/s3/s3iface + - service/s3/s3manager + - service/sts +- name: github.com/BurntSushi/toml + version: b26d9c308763d68093482582cea63d69be07a0f0 +- name: github.com/facebookgo/atomicfile + version: 2de1f203e7d5e386a6833233882782932729f27e +- name: github.com/facebookgo/symwalk + version: 42004b9f322246749dd73ad71008b1f3160c0052 +- name: github.com/ghodss/yaml + version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee +- name: github.com/gin-contrib/sse + version: 22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae +- name: github.com/gin-gonic/gin + version: d459835d2b077e44f7c9b453505ee29881d5d12d + subpackages: + - binding + - render +- name: github.com/go-ini/ini + version: c787282c39ac1fc618827141a1f762240def08a3 +- name: github.com/gobwas/glob + version: bea32b9cd2d6f55753d94a28e959b13f0244797a + subpackages: + - compiler + - match + - syntax + - syntax/ast + - syntax/lexer + - util/runes + - util/strings +- name: github.com/golang/protobuf + version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef + subpackages: + - proto + - protoc-gen-go/descriptor + - ptypes/any + - ptypes/timestamp +- name: github.com/googleapis/gax-go + version: 2cadd475a3e966ec9b77a21afc530dbacec6d613 +- name: github.com/jmespath/go-jmespath + version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +- name: github.com/kubernetes/helm + version: bbc1f71dc03afc5f00c6ac84b9308f8ecb4f39ac +- name: github.com/Masterminds/semver + version: 517734cc7d6470c0d07130e40fd40bdeb9bcd3fd +- name: github.com/mattn/go-isatty + version: fc9e8d8ef48496124e79ae0df75490096eccf6fe +- name: github.com/spf13/pflag + version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 +- name: github.com/ugorji/go + version: 8c0409fcbb70099c748d71f714529204975f6c3f + subpackages: + - codec +- name: github.com/urfave/cli + version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1 +- name: go.uber.org/atomic + version: 4e336646b2ef9fc6e47be8e21594178f98e5ebcf +- name: go.uber.org/multierr + version: 3c4937480c32f4c13a875a1829af76c98ca3d40a +- name: go.uber.org/zap + version: 9d9d6135afe89b6fc4a05e9a8552526caba38048 + subpackages: + - buffer + - internal/bufferpool + - internal/color + - internal/exit + - zapcore +- name: golang.org/x/crypto + version: 81e90905daefcd6fd217b62423c0908922eadb30 + subpackages: + - cast5 + - openpgp + - openpgp/armor + - openpgp/clearsign + - openpgp/elgamal + - openpgp/errors + - openpgp/packet + - openpgp/s2k +- name: golang.org/x/net + version: 57efc9c3d9f91fb3277f8da1cff370539c4d3dc5 + subpackages: + - context + - context/ctxhttp + - http2 + - http2/hpack + - idna + - internal/timeseries + - lex/httplex + - trace +- name: golang.org/x/oauth2 + version: 9a379c6b3e95a790ffc43293c2a78dee0d7b6e20 + subpackages: + - google + - internal + - jws + - jwt +- name: golang.org/x/sys + version: 2d6f6f883a06fc0d5f4b14a81e4c28705ea64c15 + subpackages: + - unix +- name: golang.org/x/text + version: ac87088df8ef557f1e32cd00ed0b6fbc3f7ddafb + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: google.golang.org/api + version: fe98bfd2e89a9285ca13df4260a3ea2e66589bea + subpackages: + - gensupport + - googleapi + - googleapi/internal/uritemplates + - googleapi/transport + - internal + - iterator + - option + - storage/v1 + - transport/http +- name: google.golang.org/appengine + version: d9a072cfa7b9736e44311ef77b3e09d804bfa599 + subpackages: + - internal + - internal/app_identity + - internal/base + - internal/datastore + - internal/log + - internal/modules + - internal/remote_api + - internal/urlfetch + - urlfetch +- name: google.golang.org/genproto + version: ee236bd376b077c7a89f260c026c4735b195e459 + subpackages: + - googleapis/api/annotations + - googleapis/iam/v1 + - googleapis/rpc/status +- name: google.golang.org/grpc + version: b3ddf786825de56a4178401b7e174ee332173b66 + subpackages: + - codes + - connectivity + - credentials + - grpclb/grpc_lb_v1 + - grpclog + - internal + - keepalive + - metadata + - naming + - peer + - stats + - status + - tap + - transport +- name: gopkg.in/go-playground/validator.v8 + version: 5f1438d3fca68893a817e4a66806cea46a9e4ebf +- name: gopkg.in/yaml.v2 + version: eb3733d160e74a9c7e442f435eb3bea458e1d19f +- name: k8s.io/apimachinery + version: dc1f89aff9a7509782bde3b68824c8043a3e58cc + subpackages: + - pkg/version +- name: k8s.io/helm + version: 1dbbace83192680134c96861742cedda243fcd7e + subpackages: + - pkg/chartutil + - pkg/getter + - pkg/helm/environment + - pkg/helm/helmpath + - pkg/ignore + - pkg/plugin + - pkg/proto/hapi/chart + - pkg/proto/hapi/version + - pkg/provenance + - pkg/repo + - pkg/tlsutil + - pkg/urlutil +testImports: +- name: github.com/davecgh/go-spew + version: 782f4967f2dc4564575ca782fe2d04090b5faca8 + subpackages: + - spew +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib +- name: github.com/stretchr/testify + version: f6abca593680b2315d2075e0f5e2a9751e3f431a + subpackages: + - assert + - require + - suite diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..964a0a4 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,28 @@ +package: github.com/chartmuseum/chartmuseum + +import: +- package: github.com/gin-gonic/gin + version: v1.2 +- package: github.com/kubernetes/helm + version: v2.6.1 +- package: github.com/urfave/cli + version: v1.20.0 +- package: github.com/aws/aws-sdk-go + version: v1.10.18 +- package: go.uber.org/zap + version: v1.5.0 + +# these ones are srsly a pain in da butt... +# all needed to get cloud.google.com/go/storage to work +- package: cloud.google.com/go + version: v0.12.0 +- package: google.golang.org/grpc + version: v1.5.2 +- package: golang.org/x/net + version: 57efc9c3d9f91fb3277f8da1cff370539c4d3dc5 +- package: golang.org/x/text + version: ac87088df8ef557f1e32cd00ed0b6fbc3f7ddafb + +testImports: +- package: github.com/stretchr/testify + version: v1.1.4 diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..4cd8a0a Binary files /dev/null and b/logo.png differ diff --git a/pkg/chartmuseum/handlers.go b/pkg/chartmuseum/handlers.go new file mode 100644 index 0000000..e4c642c --- /dev/null +++ b/pkg/chartmuseum/handlers.go @@ -0,0 +1,166 @@ +package chartmuseum + +import ( + "fmt" + "strings" + + "github.com/chartmuseum/chartmuseum/pkg/repo" + + "github.com/gin-gonic/gin" +) + +var ( + objectSavedResponse = gin.H{"saved": true} + objectDeletedResponse = gin.H{"deleted": true} + notFoundErrorResponse = gin.H{"error": "not found"} + badExtensionErrorResponse = gin.H{"error": "unsupported file extension"} + alreadyExistsErrorResponse = gin.H{"error": "file already exists"} +) + +func (server *Server) getIndexFileRequestHandler(c *gin.Context) { + err := server.syncRepositoryIndex() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + c.Data(200, repo.IndexFileContentType, server.RepositoryIndex.Raw) +} + +func (server *Server) getAllChartsRequestHandler(c *gin.Context) { + err := server.syncRepositoryIndex() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + c.JSON(200, server.RepositoryIndex.Entries) +} + +func (server *Server) getChartRequestHandler(c *gin.Context) { + name := c.Param("name") + err := server.syncRepositoryIndex() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + chart := server.RepositoryIndex.Entries[name] + if chart == nil { + c.JSON(404, notFoundErrorResponse) + return + } + c.JSON(200, chart) +} + +func (server *Server) getChartVersionRequestHandler(c *gin.Context) { + name := c.Param("name") + version := c.Param("version") + if version == "latest" { + version = "" + } + err := server.syncRepositoryIndex() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + chartVersion, err := server.RepositoryIndex.Get(name, version) + if err != nil { + c.JSON(404, notFoundErrorResponse) + return + } + c.JSON(200, chartVersion) +} + +func (server *Server) deleteChartVersionRequestHandler(c *gin.Context) { + name := c.Param("name") + version := c.Param("version") + filename := repo.ChartPackageFilenameFromNameVersion(name, version) + server.Logger.Debugw("Deleting package from storage", + "package", filename, + ) + err := server.StorageBackend.DeleteObject(filename) + if err != nil { + c.JSON(404, notFoundErrorResponse) + return + } + provFilename := repo.ProvenanceFilenameFromNameVersion(name, version) + server.StorageBackend.DeleteObject(provFilename) // ignore error here, may be no prov file + c.JSON(200, objectDeletedResponse) +} + +func (server *Server) getStorageObjectRequestHandler(c *gin.Context) { + filename := c.Param("filename") + isChartPackage := strings.HasSuffix(filename, repo.ChartPackageFileExtension) + isProvenanceFile := strings.HasSuffix(filename, repo.ProvenanceFileExtension) + if !isChartPackage && !isProvenanceFile { + c.JSON(500, badExtensionErrorResponse) + return + } + object, err := server.StorageBackend.GetObject(filename) + if err != nil { + c.JSON(404, notFoundErrorResponse) + return + } + if isProvenanceFile { + c.Data(200, repo.ProvenanceFileContentType, object.Content) + return + } + c.Data(200, repo.ChartPackageContentType, object.Content) +} + +func (server *Server) postPackageRequestHandler(c *gin.Context) { + content, err := c.GetRawData() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + filename, err := repo.ChartPackageFilenameFromContent(content) + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + _, err = server.StorageBackend.GetObject(filename) + if err == nil { + c.JSON(500, alreadyExistsErrorResponse) + return + } + server.Logger.Debugw("Adding package to storage", + "package", filename, + ) + err = server.StorageBackend.PutObject(filename, content) + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + c.JSON(201, objectSavedResponse) +} + +func (server *Server) postProvenanceFileRequestHandler(c *gin.Context) { + content, err := c.GetRawData() + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + filename, err := repo.ProvenanceFilenameFromContent(content) + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + _, err = server.StorageBackend.GetObject(filename) + if err == nil { + c.JSON(500, alreadyExistsErrorResponse) + return + } + server.Logger.Debugw("Adding provenance file to storage", + "provenance_file", filename, + ) + err = server.StorageBackend.PutObject(filename, content) + if err != nil { + c.JSON(500, errorResponse(err)) + return + } + c.JSON(201, objectSavedResponse) +} + +func errorResponse(err error) map[string]interface{} { + errResp := gin.H{"error": fmt.Sprintf("%s", err)} + return errResp +} diff --git a/pkg/chartmuseum/routes.go b/pkg/chartmuseum/routes.go new file mode 100644 index 0000000..2434e38 --- /dev/null +++ b/pkg/chartmuseum/routes.go @@ -0,0 +1,15 @@ +package chartmuseum + +func (server *Server) setRoutes() { + // Helm Chart Repository + server.Router.GET("/index.yaml", server.getIndexFileRequestHandler) + server.Router.GET("/charts/:filename", server.getStorageObjectRequestHandler) + + // Chart Manipulation + server.Router.GET("/api/charts", server.getAllChartsRequestHandler) + server.Router.POST("/api/charts", server.postPackageRequestHandler) + server.Router.POST("/api/prov", server.postProvenanceFileRequestHandler) + server.Router.GET("/api/charts/:name", server.getChartRequestHandler) + server.Router.GET("/api/charts/:name/:version", server.getChartVersionRequestHandler) + server.Router.DELETE("/api/charts/:name/:version", server.deleteChartVersionRequestHandler) +} diff --git a/pkg/chartmuseum/server.go b/pkg/chartmuseum/server.go new file mode 100644 index 0000000..3425846 --- /dev/null +++ b/pkg/chartmuseum/server.go @@ -0,0 +1,287 @@ +package chartmuseum + +import ( + "fmt" + "sync" + "time" + + "github.com/chartmuseum/chartmuseum/pkg/repo" + "github.com/chartmuseum/chartmuseum/pkg/storage" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + helm_repo "k8s.io/helm/pkg/repo" +) + +type ( + // Logger handles all logging from application + Logger struct { + *zap.SugaredLogger + } + + // Router handles all incoming HTTP requests + Router struct { + *gin.Engine + } + + // Server contains a Logger, Router, storage backend and object cache + Server struct { + Logger *Logger + Router *Router + RepositoryIndex *repo.Index + StorageBackend storage.Backend + StorageCache []storage.Object + StorageCacheLock *sync.Mutex + } + + // ServerOptions are options for constructing a Server + ServerOptions struct { + StorageBackend storage.Backend + LogJSON bool + Debug bool + } +) + +// NewLogger creates a new Logger instance +func NewLogger(json bool, debug bool) (*Logger, error) { + config := zap.NewDevelopmentConfig() + config.DisableStacktrace = true + config.Development = false + if json { + config.Encoding = "json" + } else { + config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + } + if !debug { + config.Level = zap.NewAtomicLevelAt(zap.InfoLevel) + } + logger, err := config.Build() + if err != nil { + return new(Logger), err + } + defer logger.Sync() + return &Logger{logger.Sugar()}, nil +} + +// NewRouter creates a new Router instance +func NewRouter(logger *Logger) *Router { + gin.SetMode(gin.ReleaseMode) + engine := gin.New() + engine.Use(loggingMiddleware(logger), gin.Recovery()) + return &Router{engine} +} + +// NewServer creates a new Server instance +func NewServer(options ServerOptions) (*Server, error) { + logger, err := NewLogger(options.LogJSON, options.Debug) + if err != nil { + return new(Server), nil + } + + router := NewRouter(logger) + + server := &Server{ + Logger: logger, + Router: router, + RepositoryIndex: repo.NewIndex(), + StorageBackend: options.StorageBackend, + StorageCache: []storage.Object{}, + StorageCacheLock: &sync.Mutex{}, + } + + server.setRoutes() + + err = server.regenerateRepositoryIndex() + return server, err +} + +// Listen starts server on a given port +func (server *Server) Listen(port int) { + server.Logger.Infow("Starting ChartMuseum", + "port", port, + ) + server.Logger.Fatal(server.Router.Run(fmt.Sprintf(":%d", port))) +} + +func loggingMiddleware(logger *Logger) gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + + msg := "Request served" + status := c.Writer.Status() + + meta := []interface{}{ + "path", c.Request.URL.Path, + "comment", c.Errors.ByType(gin.ErrorTypePrivate).String(), + "latency", time.Now().Sub(start), + "clientIP", c.ClientIP(), + "method", c.Request.Method, + "statusCode", status, + } + + switch { + case status == 200 || status == 201: + logger.Infow(msg, meta...) + case status == 404: + logger.Warnw(msg, meta...) + default: + logger.Errorw(msg, meta...) + } + } +} + +func (server *Server) syncRepositoryIndex() error { + _, diff, err := server.listObjectsGetDiff() + if err != nil { + return err + } + if !diff.Change { + return nil + } + err = server.regenerateRepositoryIndex() + return err +} + +func (server *Server) listObjectsGetDiff() ([]storage.Object, storage.ObjectSliceDiff, error) { + allObjects, err := server.StorageBackend.ListObjects() + if err != nil { + return []storage.Object{}, storage.ObjectSliceDiff{}, err + } + + // filter out storage objects that dont have extension used for chart packages (.tgz) + filteredObjects := []storage.Object{} + for _, object := range allObjects { + if object.HasExtension(repo.ChartPackageFileExtension) { + filteredObjects = append(filteredObjects, object) + } + } + + diff := storage.GetObjectSliceDiff(server.StorageCache, filteredObjects) + return filteredObjects, diff, nil +} + +func (server *Server) regenerateRepositoryIndex() error { + server.Logger.Debugw("Acquiring storage cache lock") + server.StorageCacheLock.Lock() + server.Logger.Debugw("Storage cache lock acquired") + defer func() { + server.Logger.Debugw("Releasing storage cache lock") + server.StorageCacheLock.Unlock() + }() + + objects, diff, err := server.listObjectsGetDiff() + if err != nil { + return err + } + + index := &repo.Index{IndexFile: server.RepositoryIndex.IndexFile, Raw: server.RepositoryIndex.Raw} + + for _, object := range diff.Removed { + err := server.removeIndexObject(index, object) + if err != nil { + return err + } + } + + for _, object := range diff.Updated { + err := server.updateIndexObject(index, object) + if err != nil { + return err + } + } + + // Parallelize retrieval of added objects to improve startup speed + var wg sync.WaitGroup + wg.Add(len(diff.Added)) + for _, object := range diff.Added { + go func(o storage.Object) { + defer wg.Done() + if err == nil { + if e := server.addIndexObject(index, o); e != nil { + err = e + } + } + }(object) + } + wg.Wait() + if err != nil { + return err + } + + server.Logger.Debug("Regenerating index.yaml") + err = index.Regenerate() + if err != nil { + return err + } + + server.RepositoryIndex = index + server.StorageCache = objects + return nil +} + +func (server *Server) removeIndexObject(index *repo.Index, object storage.Object) error { + chartVersion, err := server.getObjectChartVersion(object, false) + if err != nil { + return server.checkInvalidChartPackageError(object, err, "removed") + } + server.Logger.Debugw("Removing chart from index", + "name", chartVersion.Name, + "version", chartVersion.Version, + ) + index.RemoveEntry(chartVersion) + return nil +} + +func (server *Server) updateIndexObject(index *repo.Index, object storage.Object) error { + chartVersion, err := server.getObjectChartVersion(object, true) + if err != nil { + return server.checkInvalidChartPackageError(object, err, "updated") + } + server.Logger.Debugw("Updating chart in index", + "name", chartVersion.Name, + "version", chartVersion.Version, + ) + index.UpdateEntry(chartVersion) + return nil +} + +func (server *Server) addIndexObject(index *repo.Index, object storage.Object) error { + chartVersion, err := server.getObjectChartVersion(object, true) + if err != nil { + return server.checkInvalidChartPackageError(object, err, "added") + } + server.Logger.Debugw("Adding chart to index", + "name", chartVersion.Name, + "version", chartVersion.Version, + ) + index.AddEntry(chartVersion) + return nil +} + +func (server *Server) getObjectChartVersion(object storage.Object, load bool) (*helm_repo.ChartVersion, error) { + if load { + var err error + object, err = server.StorageBackend.GetObject(object.Path) + if err != nil { + return new(helm_repo.ChartVersion), err + } + if len(object.Content) == 0 { + return new(helm_repo.ChartVersion), repo.ErrorInvalidChartPackage + } + } + chartVersion, err := repo.ChartVersionFromStorageObject(object) + return chartVersion, err +} + +func (server *Server) checkInvalidChartPackageError(object storage.Object, err error, action string) error { + if err == repo.ErrorInvalidChartPackage { + server.Logger.Warnw("Invalid package in storage", + "action", action, + "package", object.Path, + ) + return nil + } + return err +} diff --git a/pkg/chartmuseum/server_test.go b/pkg/chartmuseum/server_test.go new file mode 100644 index 0000000..329ca7f --- /dev/null +++ b/pkg/chartmuseum/server_test.go @@ -0,0 +1,241 @@ +package chartmuseum + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + pathutil "path" + "testing" + "time" + + "github.com/chartmuseum/chartmuseum/pkg/storage" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/suite" +) + +var testTarballPath = "../../testdata/charts/mychart/mychart-0.1.0.tgz" +var testProvfilePath = "../../testdata/charts/mychart/mychart-0.1.0.tgz.prov" + +type ServerTestSuite struct { + suite.Suite + Server *Server + BrokenServer *Server + TempDirectory string + BrokenTempDirectory string + TestTarballFilename string + TestProvfileFilename string +} + +func (suite *ServerTestSuite) doRequest(broken bool, method string, urlStr string, body io.Reader) gin.ResponseWriter { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request, _ = http.NewRequest(method, urlStr, body) + if broken { + suite.BrokenServer.Router.HandleContext(c) + } else { + suite.Server.Router.HandleContext(c) + } + return c.Writer +} + +func (suite *ServerTestSuite) SetupSuite() { + srcFileTarball, err := os.Open(testTarballPath) + suite.Nil(err, "no error opening test tarball") + defer srcFileTarball.Close() + + srcFileProvfile, err := os.Open(testTarballPath) + suite.Nil(err, "no error opening test provfile") + defer srcFileProvfile.Close() + + timestamp := time.Now().Format("20060102150405") + suite.TempDirectory = fmt.Sprintf("../../.test/chartmuseum-server/%s", timestamp) + + backend := storage.Backend(storage.NewLocalFilesystemBackend(suite.TempDirectory)) + + server, err := NewServer(ServerOptions{backend, false, false}) + suite.NotNil(server) + suite.Nil(err, "no error creating new server, logJson=false, debug=false") + + server, err = NewServer(ServerOptions{backend, true, true}) + suite.NotNil(server) + suite.Nil(err, "no error creating new server, logJson=true, debug=true") + + server, err = NewServer(ServerOptions{backend, false, true}) + suite.Nil(err, "no error creating new server, logJson=false, debug=true") + + suite.Server = server + + suite.TestTarballFilename = pathutil.Join(suite.TempDirectory, "mychart-0.1.0.tgz") + destFileTarball, err := os.Create(suite.TestTarballFilename) + suite.Nil(err, "no error creating new tarball in temp dir") + defer destFileTarball.Close() + + _, err = io.Copy(destFileTarball, srcFileTarball) + suite.Nil(err, "no error copying test testball to temp tarball") + + err = destFileTarball.Sync() + suite.Nil(err, "no error syncing temp tarball") + + suite.TestProvfileFilename = pathutil.Join(suite.TempDirectory, "mychart-0.1.0.tgz.prov") + destFileProvfile, err := os.Create(suite.TestProvfileFilename) + suite.Nil(err, "no error creating new provenance file in temp dir") + defer destFileProvfile.Close() + + _, err = io.Copy(destFileProvfile, srcFileProvfile) + suite.Nil(err, "no error copying test provenance file to temp tarball") + + err = destFileProvfile.Sync() + suite.Nil(err, "no error syncing temp provenance file") + + suite.BrokenTempDirectory = fmt.Sprintf("../../.test/chartmuseum-server/%s-broken", timestamp) + defer os.RemoveAll(suite.BrokenTempDirectory) + + brokenBackend := storage.Backend(storage.NewLocalFilesystemBackend(suite.BrokenTempDirectory)) + brokenServer, err := NewServer(ServerOptions{brokenBackend, false, true}) + suite.Nil(err, "no error creating new server, logJson=false, debug=true") + + suite.BrokenServer = brokenServer +} + +func (suite *ServerTestSuite) TearDownSuite() { + err := os.RemoveAll(suite.TempDirectory) + suite.Nil(err, "no error deleting temp directory for local storage") +} + +func (suite *ServerTestSuite) TestRegenerateRepositoryIndex() { + err := suite.Server.regenerateRepositoryIndex() + suite.Nil(err, "no error regenerating repo index") + + newtime := time.Now().Add(1 * time.Hour) + err = os.Chtimes(suite.TestTarballFilename, newtime, newtime) + suite.Nil(err, "no error changing modtime on temp file") + err = suite.Server.regenerateRepositoryIndex() + suite.Nil(err, "no error regenerating repo index with tarball updated") + + brokenTarballFilename := pathutil.Join(suite.TempDirectory, "brokenchart.tgz") + destFile, err := os.Create(brokenTarballFilename) + suite.Nil(err, "no error creating new broken tarball in temp dir") + defer destFile.Close() + err = suite.Server.regenerateRepositoryIndex() + suite.Nil(err, "error not returned with broken tarball added") + + err = os.Chtimes(brokenTarballFilename, newtime, newtime) + suite.Nil(err, "no error changing modtime on broken tarball") + err = suite.Server.regenerateRepositoryIndex() + suite.Nil(err, "error not returned with broken tarball updated") + + err = os.Remove(brokenTarballFilename) + suite.Nil(err, "no error removing broken tarball") + err = suite.Server.regenerateRepositoryIndex() + suite.Nil(err, "error not returned with broken tarball removed") +} + +func (suite *ServerTestSuite) TestRoutes() { + var body io.Reader + var res gin.ResponseWriter + + // GET /charts/ + res = suite.doRequest(false, "GET", "/charts/mychart-0.1.0.tgz", nil) + suite.Equal(200, res.Status(), "200 GET /charts/mychart-0.1.0.tgz") + + res = suite.doRequest(false, "GET", "/charts/mychart-0.1.0.tgz.prov", nil) + suite.Equal(200, res.Status(), "200 GET /charts/mychart-0.1.0.tgz.prov") + + res = suite.doRequest(false, "GET", "/charts/fakechart-0.1.0.tgz", nil) + suite.Equal(404, res.Status(), "404 GET /charts/fakechart-0.1.0.tgz") + + res = suite.doRequest(false, "GET", "/charts/fakechart-0.1.0.tgz.prov", nil) + suite.Equal(404, res.Status(), "404 GET /charts/fakechart-0.1.0.tgz.prov") + + res = suite.doRequest(false, "GET", "/charts/fakechart-0.1.0.bad", nil) + suite.Equal(500, res.Status(), "500 GET /charts/fakechart-0.1.0.bad") + + // GET /api/charts + res = suite.doRequest(false, "GET", "/api/charts", nil) + suite.Equal(200, res.Status(), "200 GET /api/charts") + + res = suite.doRequest(true, "GET", "/api/charts", nil) + suite.Equal(500, res.Status(), "500 GET /api/charts") + + // GET /api/charts/ + res = suite.doRequest(false, "GET", "/api/charts/mychart", nil) + suite.Equal(200, res.Status(), "200 GET /api/charts/mychart") + + res = suite.doRequest(false, "GET", "/api/charts/fakechart", nil) + suite.Equal(404, res.Status(), "404 GET /api/charts/fakechart") + + res = suite.doRequest(true, "GET", "/api/charts/mychart", nil) + suite.Equal(500, res.Status(), "500 GET /api/charts/mychart") + + // GET /api/charts// + res = suite.doRequest(false, "GET", "/api/charts/mychart/0.1.0", nil) + suite.Equal(200, res.Status(), "200 GET /api/charts/mychart/0.1.0") + + res = suite.doRequest(false, "GET", "/api/charts/mychart/latest", nil) + suite.Equal(200, res.Status(), "200 GET /api/charts/mychart/latest") + + res = suite.doRequest(false, "GET", "/api/charts/mychart/0.0.0", nil) + suite.Equal(404, res.Status(), "404 GET /api/charts/mychart/0.0.0") + + res = suite.doRequest(false, "GET", "/api/charts/fakechart/0.1.0", nil) + suite.Equal(404, res.Status(), "404 GET /api/charts/fakechart/0.1.0") + + res = suite.doRequest(true, "GET", "/api/charts/mychart/0.1.0", nil) + suite.Equal(500, res.Status(), "500 GET /api/charts/mychart/0.1.0") + + // DELETE /api/charts// + res = suite.doRequest(false, "DELETE", "/api/charts/mychart/0.1.0", nil) + suite.Equal(200, res.Status(), "200 DELETE /api/charts/mychart/0.1.0") + + res = suite.doRequest(false, "DELETE", "/api/charts/mychart/0.1.0", nil) + suite.Equal(404, res.Status(), "404 DELETE /api/charts/mychart/0.1.0") + + // GET /index.yaml + res = suite.doRequest(false, "GET", "/index.yaml", nil) + suite.Equal(200, res.Status(), "200 GET /index.yaml") + + res = suite.doRequest(true, "GET", "/index.yaml", nil) + suite.Equal(500, res.Status(), "500 GET /index.yaml") + + // POST /api/charts + body = bytes.NewBuffer([]byte{}) + res = suite.doRequest(false, "POST", "/api/charts", body) + suite.Equal(500, res.Status(), "500 POST /api/charts") + + // POST /api/prov + body = bytes.NewBuffer([]byte{}) + res = suite.doRequest(false, "POST", "/api/prov", body) + suite.Equal(500, res.Status(), "500 POST /api/prov") + + // POST /api/charts + content, err := ioutil.ReadFile(testTarballPath) + suite.Nil(err, "no error opening test tarball") + + body = bytes.NewBuffer(content) + res = suite.doRequest(false, "POST", "/api/charts", body) + suite.Equal(201, res.Status(), "201 POST /api/charts") + + body = bytes.NewBuffer(content) + res = suite.doRequest(false, "POST", "/api/charts", body) + suite.Equal(500, res.Status(), "500 POST /api/charts") + + // POST /api/prov + content, err = ioutil.ReadFile(testProvfilePath) + suite.Nil(err, "no error opening test provenance file") + + body = bytes.NewBuffer(content) + res = suite.doRequest(false, "POST", "/api/prov", body) + suite.Equal(201, res.Status(), "201 POST /api/prov") + + body = bytes.NewBuffer(content) + res = suite.doRequest(false, "POST", "/api/prov", body) + suite.Equal(500, res.Status(), "500 POST /api/prov") +} + +func TestServerTestSuite(t *testing.T) { + suite.Run(t, new(ServerTestSuite)) +} diff --git a/pkg/repo/chart.go b/pkg/repo/chart.go new file mode 100644 index 0000000..1a424b7 --- /dev/null +++ b/pkg/repo/chart.go @@ -0,0 +1,84 @@ +package repo + +import ( + "bytes" + "errors" + "fmt" + pathutil "path" + "strings" + + "github.com/chartmuseum/chartmuseum/pkg/storage" + + "k8s.io/helm/pkg/chartutil" + helm_chart "k8s.io/helm/pkg/proto/hapi/chart" + helm_repo "k8s.io/helm/pkg/repo" +) + +var ( + // ChartPackageFileExtension is the file extension used for chart packages + ChartPackageFileExtension = "tgz" + + // ChartPackageContentType is the http content-type header for chart packages + ChartPackageContentType = "application/x-tar" + + // ErrorInvalidChartPackage is raised when a chart package is invalid + ErrorInvalidChartPackage = errors.New("invalid chart package") +) + +// ChartPackageFilenameFromNameVersion returns a chart filename from a name and version +func ChartPackageFilenameFromNameVersion(name string, version string) string { + filename := fmt.Sprintf("%s-%s.%s", name, version, ChartPackageFileExtension) + return filename +} + +// ChartPackageFilenameFromContent returns a chart filename from binary content +func ChartPackageFilenameFromContent(content []byte) (string, error) { + chart, err := chartFromContent(content) + if err != nil { + return "", err + } + meta := chart.Metadata + filename := fmt.Sprintf("%s-%s.%s", meta.Name, meta.Version, ChartPackageFileExtension) + return filename, nil +} + +// ChartVersionFromStorageObject returns a chart version from a storage object +func ChartVersionFromStorageObject(object storage.Object) (*helm_repo.ChartVersion, error) { + if len(object.Content) == 0 { + chartVersion := emptyChartVersionFromPackageFilename(object.Path) + if chartVersion.Name == "" || chartVersion.Version == "" { + return chartVersion, ErrorInvalidChartPackage + } + return chartVersion, nil + } + chart, err := chartFromContent(object.Content) + if err != nil { + return new(helm_repo.ChartVersion), ErrorInvalidChartPackage + } + digest, err := provenanceDigestFromContent(object.Content) + if err != nil { + return new(helm_repo.ChartVersion), err + } + chartVersion := &helm_repo.ChartVersion{ + URLs: []string{fmt.Sprintf("charts/%s", object.Path)}, + Metadata: chart.Metadata, + Digest: digest, + Created: object.LastModified, + } + return chartVersion, nil +} + +func chartFromContent(content []byte) (*helm_chart.Chart, error) { + chart, err := chartutil.LoadArchive(bytes.NewBuffer(content)) + return chart, err +} + +func emptyChartVersionFromPackageFilename(filename string) *helm_repo.ChartVersion { + noExt := strings.TrimSuffix(pathutil.Base(filename), fmt.Sprintf(".%s", ChartPackageFileExtension)) + tmp := strings.Split(noExt, "-") + lastIndex := len(tmp) - 1 + name := strings.Join(tmp[:lastIndex], "-") + version := tmp[lastIndex] + metadata := &helm_chart.Metadata{Name: name, Version: version} + return &helm_repo.ChartVersion{Metadata: metadata} +} diff --git a/pkg/repo/chart_test.go b/pkg/repo/chart_test.go new file mode 100644 index 0000000..cdb6f31 --- /dev/null +++ b/pkg/repo/chart_test.go @@ -0,0 +1,72 @@ +package repo + +import ( + "io/ioutil" + "testing" + "time" + + "github.com/chartmuseum/chartmuseum/pkg/storage" + + "github.com/stretchr/testify/suite" +) + +type ChartTestSuite struct { + suite.Suite + TarballContent []byte +} + +func (suite *ChartTestSuite) SetupSuite() { + tarballPath := "../../testdata/charts/mychart/mychart-0.1.0.tgz" + content, err := ioutil.ReadFile(tarballPath) + suite.Nil(err, "no error reading test tarball") + suite.TarballContent = content +} + +func (suite *ChartTestSuite) TestChartPackageFilenameFromNameVersion() { + filename := ChartPackageFilenameFromNameVersion("mychart", "2.3.4") + suite.Equal("mychart-2.3.4.tgz", filename, "filename as expected") +} + +func (suite *ChartTestSuite) TestChartVersionFromStorageObject() { + object := storage.Object{ + Path: "mychart-2.3.4.tgz", + Content: []byte{}, + LastModified: time.Now(), + } + chartVersion, err := ChartVersionFromStorageObject(object) + suite.Nil(err, "no error creating ChartVersion from storage.Object") + suite.Equal("mychart", chartVersion.Name, "chart name as expected") + suite.Equal("2.3.4", chartVersion.Version, "chart version as expected") + + object.Content = suite.TarballContent + chartVersion, err = ChartVersionFromStorageObject(object) + suite.Nil(err) + suite.Equal("mychart", chartVersion.Name, "chart name as expected") + suite.Equal("0.1.0", chartVersion.Version, "chart version as expected") + + object.Content = []byte("this should create an error") + _, err = ChartVersionFromStorageObject(object) + suite.NotNil(err, "error creating ChartVersion from storage.Object with bad content") + + brokenObject := storage.Object{ + Path: "brokenchart.tgz", + Content: []byte{}, + LastModified: time.Now(), + } + _, err = ChartVersionFromStorageObject(brokenObject) + suite.Equal(err, ErrorInvalidChartPackage, "error creating ChartVersion from storage.Object with bad content") +} + +func (suite *ChartTestSuite) TestChartPackageFilenameFromContent() { + filename, err := ChartPackageFilenameFromContent([]byte{}) + suite.NotNil(err, "error getting tarball filename with empty byte array") + suite.Equal("", filename, "filename blank with empty byte array") + + filename, err = ChartPackageFilenameFromContent(suite.TarballContent) + suite.Nil(err, "no error getting filename from test tarball content") + suite.Equal("mychart-0.1.0.tgz", filename, "chart tarball filename as expected") +} + +func TestChartTestSuite(t *testing.T) { + suite.Run(t, new(ChartTestSuite)) +} diff --git a/pkg/repo/index.go b/pkg/repo/index.go new file mode 100644 index 0000000..6ca28a3 --- /dev/null +++ b/pkg/repo/index.go @@ -0,0 +1,82 @@ +package repo + +import ( + "time" + + "github.com/ghodss/yaml" + + helm_repo "k8s.io/helm/pkg/repo" +) + +var ( + // IndexFileContentType is the http content-type header for index.yaml + IndexFileContentType = "application/x-yaml" +) + +// Index represents the repository index (index.yaml) +type Index struct { + *helm_repo.IndexFile + Raw []byte +} + +// NewIndex creates a new instance of Index +func NewIndex() *Index { + index := Index{&helm_repo.IndexFile{}, []byte{}} + index.Entries = map[string]helm_repo.ChartVersions{} + index.APIVersion = helm_repo.APIVersionV1 + return &index +} + +// Regenerate sorts entries in index file and sets current time for generated key +func (index *Index) Regenerate() error { + index.SortEntries() + index.Generated = time.Now().Round(time.Second) + raw, err := yaml.Marshal(index.IndexFile) + if err != nil { + return err + } + index.Raw = raw + return nil +} + +// RemoveEntry removes a chart version from index +func (index *Index) RemoveEntry(chartVersion *helm_repo.ChartVersion) { + for k := range index.Entries { + if k == chartVersion.Name { + for i, cv := range index.Entries[chartVersion.Name] { + if cv.Version == chartVersion.Version { + index.Entries[chartVersion.Name] = append(index.Entries[chartVersion.Name][:i], + index.Entries[chartVersion.Name][i+1:]...) + if len(index.Entries[chartVersion.Name]) == 0 { + delete(index.Entries, chartVersion.Name) + } + break + } + } + break + } + } +} + +// AddEntry adds a chart version to index +func (index *Index) AddEntry(chartVersion *helm_repo.ChartVersion) { + if _, ok := index.Entries[chartVersion.Name]; !ok { + index.Entries[chartVersion.Name] = helm_repo.ChartVersions{} + } + index.Entries[chartVersion.Name] = append(index.Entries[chartVersion.Name], chartVersion) +} + +// UpdateEntry updates a chart version in index +func (index *Index) UpdateEntry(chartVersion *helm_repo.ChartVersion) { + for k := range index.Entries { + if k == chartVersion.Name { + for i, cv := range index.Entries[chartVersion.Name] { + if cv.Version == chartVersion.Version { + index.Entries[chartVersion.Name][i] = chartVersion + break + } + } + break + } + } +} diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go new file mode 100644 index 0000000..b2247c4 --- /dev/null +++ b/pkg/repo/index_test.go @@ -0,0 +1,76 @@ +package repo + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "k8s.io/helm/pkg/proto/hapi/chart" + helm_repo "k8s.io/helm/pkg/repo" +) + +type IndexTestSuite struct { + suite.Suite + Index *Index +} + +func getChartVersion(name string, patch int, created time.Time) *helm_repo.ChartVersion { + version := fmt.Sprintf("1.0.%d", patch) + metadata := chart.Metadata{ + Name: name, + Version: version, + } + chartVersion := helm_repo.ChartVersion{ + Metadata: &metadata, + URLs: []string{}, + Created: created, + Removed: false, + Digest: "", + } + return &chartVersion +} + +func (suite *IndexTestSuite) SetupSuite() { + suite.Index = NewIndex() + now := time.Now() + for _, name := range []string{"a", "b", "c"} { + for i := 0; i < 10; i++ { + chartVersion := getChartVersion(name, i, now) + suite.Index.AddEntry(chartVersion) + } + } + chartVersion := getChartVersion("d", 0, now) + suite.Index.AddEntry(chartVersion) +} + +func (suite *IndexTestSuite) TestRegenerate() { + err := suite.Index.Regenerate() + suite.Nil(err) +} + +func (suite *IndexTestSuite) TestUpdate() { + now := time.Now() + for _, name := range []string{"a", "b", "c"} { + for i := 0; i < 5; i++ { + chartVersion := getChartVersion(name, i, now) + suite.Index.UpdateEntry(chartVersion) + } + } +} + +func (suite *IndexTestSuite) TestRemove() { + now := time.Now() + for _, name := range []string{"a", "b", "c"} { + for i := 5; i < 10; i++ { + chartVersion := getChartVersion(name, i, now) + suite.Index.RemoveEntry(chartVersion) + } + } + chartVersion := getChartVersion("d", 0, now) + suite.Index.RemoveEntry(chartVersion) +} + +func TestIndexTestSuite(t *testing.T) { + suite.Run(t, new(IndexTestSuite)) +} diff --git a/pkg/repo/provenance.go b/pkg/repo/provenance.go new file mode 100644 index 0000000..804a2b0 --- /dev/null +++ b/pkg/repo/provenance.go @@ -0,0 +1,49 @@ +package repo + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "k8s.io/helm/pkg/provenance" + "regexp" +) + +var ( + // ProvenanceFileExtension is the file extension used for provenance files + ProvenanceFileExtension = "tgz.prov" + + // ProvenanceFileContentType is the http content-type header for provenance files + ProvenanceFileContentType = "application/pgp-signature" + + // ErrorInvalidProvenanceFile is raised when a provenance file is invalid + ErrorInvalidProvenanceFile = errors.New("invalid provenance file") +) + +// ProvenanceFilenameFromNameVersion returns a provenance filename from a name and version +func ProvenanceFilenameFromNameVersion(name string, version string) string { + filename := fmt.Sprintf("%s-%s.%s", name, version, ProvenanceFileExtension) + return filename +} + +// ProvenanceFilenameFromContent returns a provenance filename from binary content +func ProvenanceFilenameFromContent(content []byte) (string, error) { + contentStr := string(content[:]) + + hasPGPBegin := strings.HasPrefix(contentStr, "-----BEGIN PGP SIGNED MESSAGE-----") + nameMatch := regexp.MustCompile("name:[ *](.+)").FindStringSubmatch(contentStr) + versionMatch := regexp.MustCompile("version:[ *](.+)").FindStringSubmatch(contentStr) + + if !hasPGPBegin || len(nameMatch) != 2 || len(versionMatch) != 2 { + return "", ErrorInvalidProvenanceFile + } + + filename := ProvenanceFilenameFromNameVersion(nameMatch[1], versionMatch[1]) + return filename, nil +} + +func provenanceDigestFromContent(content []byte) (string, error) { + digest, err := provenance.Digest(bytes.NewBuffer(content)) + return digest, err +} diff --git a/pkg/repo/provenance_test.go b/pkg/repo/provenance_test.go new file mode 100644 index 0000000..ae49b38 --- /dev/null +++ b/pkg/repo/provenance_test.go @@ -0,0 +1,55 @@ +package repo + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type ProvenanceTestSuite struct { + suite.Suite +} + +func (suite *ProvenanceTestSuite) TestProvenanceFileFilenameFromContent() { + goodContent := []byte(`-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +name: mychart +version: 0.1.0 + +... +files: + mychart-0.1.0.tgz: sha256:5c824605d676f5244aaf70d889f4e58f953308c426f2fa8f970e8fd580eaf363 +-----BEGIN PGP SIGNATURE----- + +wsBcBAEBCgAQBQJZuxVACRCEO7+YH8GHYgAAtVMIAEIKSyWH9hb3y/ck6Dwg2Y6v +6i0kP3L9iCyyTp64XJYiuipdhUO/XK0CxRcLqLa0I5qu658XeU/Qxwb1GTgPoP52 +BCyiJVOY5aXl0SJa+jXHliDak7fgZjUHCtp1HBEKX2uRrx57tTkIjZr7pitt/OwI +bRz9OXHQe9+fhtAZo5DPtMd53UQ2uRc7xft9HxnwlDEWrBfH6CUNlhbdtKRR5n0s +FUyR0Eszw/x3No0DdPuH3fo0ShamW9eOFnXIgWqvaeSJthTC5WO5mlSGNEunJKft +HjQLzdEWppyu55ZS6/oIJdVC2GjUa/PZmKkhYwsMvaWYv+jZWFfhZn8fPYEF0qI= +=/cXn +-----END PGP SIGNATURE-----`) + badContentNoBeginPGP := []byte("badbadverybad") + badContentNoChartName := []byte(`-----BEGIN PGP SIGNED MESSAGE----- +version: 0.1.0`) + badContentNoChartVersion := []byte(`-----BEGIN PGP SIGNED MESSAGE----- +name: mychart`) + + filename, err := ProvenanceFilenameFromContent(goodContent) + suite.Nil(err, "no error getting filename from good content") + suite.Equal("mychart-0.1.0.tgz.prov", filename, "filename generated from good content") + + _, err = ProvenanceFilenameFromContent(badContentNoBeginPGP) + suite.Equal(ErrorInvalidProvenanceFile, err, "ErrorInvalidProvenanceFile from bad content, no begin pgp") + + _, err = ProvenanceFilenameFromContent(badContentNoChartName) + suite.Equal(ErrorInvalidProvenanceFile, err, "ErrorInvalidProvenanceFile from bad content, no name") + + _, err = ProvenanceFilenameFromContent(badContentNoChartVersion) + suite.Equal(ErrorInvalidProvenanceFile, err, "ErrorInvalidProvenanceFile from bad content, no version") +} + +func TestProvenanceTestSuite(t *testing.T) { + suite.Run(t, new(ProvenanceTestSuite)) +} diff --git a/pkg/storage/amazon.go b/pkg/storage/amazon.go new file mode 100644 index 0000000..08901ff --- /dev/null +++ b/pkg/storage/amazon.go @@ -0,0 +1,111 @@ +package storage + +import ( + "bytes" + "io/ioutil" + pathutil "path" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +// AmazonS3Backend is a storage backend for Amazon S3 +type AmazonS3Backend struct { + Bucket string + Client *s3.S3 + Downloader *s3manager.Downloader + Prefix string + Uploader *s3manager.Uploader +} + +// NewAmazonS3Backend creates a new instance of AmazonS3Backend +func NewAmazonS3Backend(bucket string, prefix string, region string) *AmazonS3Backend { + service := s3.New(session.New(), &aws.Config{ + Region: aws.String(region), + }) + b := &AmazonS3Backend{ + Bucket: bucket, + Client: service, + Downloader: s3manager.NewDownloaderWithClient(service), + Prefix: cleanPrefix(prefix), + Uploader: s3manager.NewUploaderWithClient(service), + } + return b +} + +// ListObjects lists all objects in Amazon S3 bucket, at prefix +func (b AmazonS3Backend) ListObjects() ([]Object, error) { + var objects []Object + s3Input := &s3.ListObjectsInput{ + Bucket: aws.String(b.Bucket), + Prefix: aws.String(b.Prefix), + } + for { + s3Result, err := b.Client.ListObjects(s3Input) + if err != nil { + return objects, err + } + for _, obj := range s3Result.Contents { + path := removePrefixFromObjectPath(b.Prefix, *obj.Key) + if objectPathIsInvalid(path) { + continue + } + object := Object{ + Path: path, + Content: []byte{}, + LastModified: *obj.LastModified, + } + objects = append(objects, object) + } + if !*s3Result.IsTruncated { + break + } + s3Input.Marker = s3Result.Contents[len(s3Result.Contents)-1].Key + } + return objects, nil +} + +// GetObject retrieves an object from Amazon S3 bucket, at prefix +func (b AmazonS3Backend) GetObject(path string) (Object, error) { + var object Object + object.Path = path + var content []byte + s3Input := &s3.GetObjectInput{ + Bucket: aws.String(b.Bucket), + Key: aws.String(pathutil.Join(b.Prefix, path)), + } + s3Result, err := b.Client.GetObject(s3Input) + if err != nil { + return object, err + } + content, err = ioutil.ReadAll(s3Result.Body) + if err != nil { + return object, err + } + object.Content = content + object.LastModified = *s3Result.LastModified + return object, nil +} + +// PutObject uploads an object to Amazon S3 bucket, at prefix +func (b AmazonS3Backend) PutObject(path string, content []byte) error { + s3Input := &s3manager.UploadInput{ + Bucket: aws.String(b.Bucket), + Key: aws.String(pathutil.Join(b.Prefix, path)), + Body: bytes.NewBuffer(content), + } + _, err := b.Uploader.Upload(s3Input) + return err +} + +// DeleteObject removes an object from Amazon S3 bucket, at prefix +func (b AmazonS3Backend) DeleteObject(path string) error { + s3Input := &s3.DeleteObjectInput{ + Bucket: aws.String(b.Bucket), + Key: aws.String(pathutil.Join(b.Prefix, path)), + } + _, err := b.Client.DeleteObject(s3Input) + return err +} diff --git a/pkg/storage/amazon_test.go b/pkg/storage/amazon_test.go new file mode 100644 index 0000000..e0596f8 --- /dev/null +++ b/pkg/storage/amazon_test.go @@ -0,0 +1,58 @@ +package storage + +import ( + "os" + "testing" + + "github.com/stretchr/testify/suite" +) + +type AmazonTestSuite struct { + suite.Suite + BrokenAmazonS3Backend *AmazonS3Backend + NoPrefixAmazonS3Backend *AmazonS3Backend +} + +func (suite *AmazonTestSuite) SetupSuite() { + backend := NewAmazonS3Backend("fake-bucket-cant-exist-fbce123", "", "us-east-1") + suite.BrokenAmazonS3Backend = backend + + s3Bucket := os.Getenv("TEST_STORAGE_AMAZON_BUCKET") + s3Region := os.Getenv("TEST_STORAGE_AMAZON_REGION") + backend = NewAmazonS3Backend(s3Bucket, "", s3Region) + suite.NoPrefixAmazonS3Backend = backend + + data := []byte("some object") + path := "deleteme.txt" + err := suite.NoPrefixAmazonS3Backend.PutObject(path, data) + suite.Nil(err, "no error putting deleteme.txt using AmazonS3 backend") +} + +func (suite *AmazonTestSuite) TearDownSuite() { + err := suite.NoPrefixAmazonS3Backend.DeleteObject("deleteme.txt") + suite.Nil(err, "no error deleting deleteme.txt using AmazonS3 backend") +} + +func (suite *AmazonTestSuite) TestListObjects() { + _, err := suite.BrokenAmazonS3Backend.ListObjects() + suite.NotNil(err, "cannot list objects with bad bucket") + + _, err = suite.NoPrefixAmazonS3Backend.ListObjects() + suite.Nil(err, "can list objects with good bucket, no prefix") +} + +func (suite *AmazonTestSuite) TestGetObject() { + _, err := suite.BrokenAmazonS3Backend.GetObject("this-file-cannot-possibly-exist.tgz") + suite.NotNil(err, "cannot get objects with bad bucket") +} + +func (suite *AmazonTestSuite) TestPutObject() { + err := suite.BrokenAmazonS3Backend.PutObject("this-file-will-not-upload.txt", []byte{}) + suite.NotNil(err, "cannot put objects with bad bucket") +} + +func TestAmazonStorageTestSuite(t *testing.T) { + if os.Getenv("TEST_CLOUD_STORAGE") == "1" { + suite.Run(t, new(AmazonTestSuite)) + } +} diff --git a/pkg/storage/google.go b/pkg/storage/google.go new file mode 100644 index 0000000..9077bc7 --- /dev/null +++ b/pkg/storage/google.go @@ -0,0 +1,103 @@ +package storage + +import ( + "io/ioutil" + pathutil "path" + + "cloud.google.com/go/storage" + "golang.org/x/net/context" + "google.golang.org/api/iterator" +) + +// GoogleCSBackend is a storage backend for Google Cloud Storage +type GoogleCSBackend struct { + Prefix string + Query *storage.Query + Client *storage.BucketHandle + Context context.Context +} + +// NewGoogleCSBackend creates a new instance of GoogleCSBackend +func NewGoogleCSBackend(bucket string, prefix string) *GoogleCSBackend { + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + panic(err) + } + bucketHandle := client.Bucket(bucket) + prefix = cleanPrefix(prefix) + listQuery := storage.Query{Prefix: prefix} + b := &GoogleCSBackend{ + Prefix: prefix, + Query: &listQuery, + Client: bucketHandle, + Context: ctx, + } + return b +} + +// ListObjects lists all objects in Google Cloud Storage bucket, at prefix +func (b GoogleCSBackend) ListObjects() ([]Object, error) { + var objects []Object + it := b.Client.Objects(b.Context, b.Query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return objects, err + } + path := removePrefixFromObjectPath(b.Prefix, attrs.Name) + if objectPathIsInvalid(path) { + continue + } + object := Object{ + Path: path, + Content: []byte{}, + LastModified: attrs.Updated, + } + objects = append(objects, object) + } + return objects, nil +} + +// GetObject retrieves an object from Google Cloud Storage bucket, at prefix +func (b GoogleCSBackend) GetObject(path string) (Object, error) { + var object Object + object.Path = path + objectHandle := b.Client.Object(pathutil.Join(b.Prefix, path)) + attrs, err := objectHandle.Attrs(b.Context) + if err != nil { + return object, err + } + object.LastModified = attrs.Updated + rc, err := objectHandle.NewReader(b.Context) + if err != nil { + return object, err + } + content, err := ioutil.ReadAll(rc) + rc.Close() + if err != nil { + return object, err + } + object.Content = content + return object, nil +} + +// PutObject uploads an object to Google Cloud Storage bucket, at prefix +func (b GoogleCSBackend) PutObject(path string, content []byte) error { + wc := b.Client.Object(pathutil.Join(b.Prefix, path)).NewWriter(b.Context) + _, err := wc.Write(content) + if err != nil { + return err + } + err = wc.Close() + return err +} + +// DeleteObject removes an object from Google Cloud Storage bucket, at prefix +func (b GoogleCSBackend) DeleteObject(path string) error { + err := b.Client.Object(pathutil.Join(b.Prefix, path)).Delete(b.Context) + return err +} diff --git a/pkg/storage/google_test.go b/pkg/storage/google_test.go new file mode 100644 index 0000000..33fcfc6 --- /dev/null +++ b/pkg/storage/google_test.go @@ -0,0 +1,57 @@ +package storage + +import ( + "os" + "testing" + + "github.com/stretchr/testify/suite" +) + +type GoogleTestSuite struct { + suite.Suite + BrokenGoogleCSBackend *GoogleCSBackend + NoPrefixGoogleCSBackend *GoogleCSBackend +} + +func (suite *GoogleTestSuite) SetupSuite() { + backend := NewGoogleCSBackend("fake-bucket-cant-exist-fbce123", "") + suite.BrokenGoogleCSBackend = backend + + gcsBucket := os.Getenv("TEST_STORAGE_GOOGLE_BUCKET") + backend = NewGoogleCSBackend(gcsBucket, "") + suite.NoPrefixGoogleCSBackend = backend + + data := []byte("some object") + path := "deleteme.txt" + err := suite.NoPrefixGoogleCSBackend.PutObject(path, data) + suite.Nil(err, "no error putting deleteme.txt using GoogleCS backend") +} + +func (suite *GoogleTestSuite) TearDownSuite() { + err := suite.NoPrefixGoogleCSBackend.DeleteObject("deleteme.txt") + suite.Nil(err, "no error deleting deleteme.txt using GoogleCS backend") +} + +func (suite *GoogleTestSuite) TestListObjects() { + _, err := suite.BrokenGoogleCSBackend.ListObjects() + suite.NotNil(err, "cannot list objects with bad bucket") + + _, err = suite.NoPrefixGoogleCSBackend.ListObjects() + suite.Nil(err, "can list objects with good bucket, no prefix") +} + +func (suite *GoogleTestSuite) TestGetObject() { + _, err := suite.BrokenGoogleCSBackend.GetObject("this-file-cannot-possibly-exist.tgz") + suite.NotNil(err, "cannot get objects with bad bucket") +} + +func (suite *GoogleTestSuite) TestPutObject() { + err := suite.BrokenGoogleCSBackend.PutObject("this-file-will-not-upload.txt", []byte{}) + suite.NotNil(err, "cannot put objects with bad bucket") +} + +func TestGoogleStorageTestSuite(t *testing.T) { + if os.Getenv("TEST_CLOUD_STORAGE") == "1" { + suite.Run(t, new(GoogleTestSuite)) + } +} diff --git a/pkg/storage/local.go b/pkg/storage/local.go new file mode 100644 index 0000000..60d4fae --- /dev/null +++ b/pkg/storage/local.go @@ -0,0 +1,74 @@ +package storage + +import ( + "io/ioutil" + "os" + + pathutil "path" +) + +// LocalFilesystemBackend is a storage backend for local filesystem storage +type LocalFilesystemBackend struct { + RootDirectory string +} + +// NewLocalFilesystemBackend creates a new instance of LocalFilesystemBackend +func NewLocalFilesystemBackend(rootDirectory string) *LocalFilesystemBackend { + if _, err := os.Stat(rootDirectory); os.IsNotExist(err) { + err := os.MkdirAll(rootDirectory, 0777) + if err != nil { + panic(err) + } + } + b := &LocalFilesystemBackend{RootDirectory: rootDirectory} + return b +} + +// ListObjects lists all objects in root directory (depth 1) +func (b LocalFilesystemBackend) ListObjects() ([]Object, error) { + var objects []Object + files, err := ioutil.ReadDir(b.RootDirectory) + if err != nil { + return objects, err + } + for _, f := range files { + if f.IsDir() { + continue + } + object := Object{Path: f.Name(), Content: []byte{}, LastModified: f.ModTime()} + objects = append(objects, object) + } + return objects, nil +} + +// GetObject retrieves an object from root directory +func (b LocalFilesystemBackend) GetObject(path string) (Object, error) { + var object Object + object.Path = path + fullpath := pathutil.Join(b.RootDirectory, path) + content, err := ioutil.ReadFile(fullpath) + if err != nil { + return object, err + } + object.Content = content + info, err := os.Stat(fullpath) + if err != nil { + return object, err + } + object.LastModified = info.ModTime() + return object, err +} + +// PutObject puts an object in root directory +func (b LocalFilesystemBackend) PutObject(path string, content []byte) error { + fullpath := pathutil.Join(b.RootDirectory, path) + err := ioutil.WriteFile(fullpath, content, 0644) + return err +} + +// DeleteObject removes an object from root directory +func (b LocalFilesystemBackend) DeleteObject(path string) error { + fullpath := pathutil.Join(b.RootDirectory, path) + err := os.Remove(fullpath) + return err +} diff --git a/pkg/storage/local_test.go b/pkg/storage/local_test.go new file mode 100644 index 0000000..ccfc741 --- /dev/null +++ b/pkg/storage/local_test.go @@ -0,0 +1,38 @@ +package storage + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type LocalTestSuite struct { + suite.Suite + LocalFilesystemBackend *LocalFilesystemBackend + BrokenTempDirectory string +} + +func (suite *LocalTestSuite) SetupSuite() { + timestamp := time.Now().Format("20060102150405") + suite.BrokenTempDirectory = fmt.Sprintf("../../.test/storage-local/%s-broken", timestamp) + defer os.RemoveAll(suite.BrokenTempDirectory) + backend := NewLocalFilesystemBackend(suite.BrokenTempDirectory) + suite.LocalFilesystemBackend = backend +} + +func (suite *LocalTestSuite) TestListObjects() { + _, err := suite.LocalFilesystemBackend.ListObjects() + suite.NotNil(err, "cannot list objects with bad root dir") +} + +func (suite *LocalTestSuite) TestGetObject() { + _, err := suite.LocalFilesystemBackend.GetObject("this-file-cannot-possibly-exist.tgz") + suite.NotNil(err, "cannot get objects with bad path") +} + +func TestLocalStorageTestSuite(t *testing.T) { + suite.Run(t, new(LocalTestSuite)) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..0637ce2 --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,88 @@ +package storage + +import ( + "fmt" + "path/filepath" + "strings" + "time" +) + +type ( + // Object is a generic representation of a storage object + Object struct { + Path string + Content []byte + LastModified time.Time + } + + // ObjectSliceDiff provides information on what has changed since last calling ListObjects + ObjectSliceDiff struct { + Change bool + Removed []Object + Added []Object + Updated []Object + } + + // Backend is a generic interface for storage backends + Backend interface { + ListObjects() ([]Object, error) + GetObject(path string) (Object, error) + PutObject(path string, content []byte) error + DeleteObject(path string) error + } +) + +// HasExtension determines whether or not an object contains a file extension +func (object Object) HasExtension(extension string) bool { + return filepath.Ext(object.Path) == fmt.Sprintf(".%s", extension) +} + +// GetObjectSliceDiff takes two objects slices and returns an ObjectSliceDiff +func GetObjectSliceDiff(os1 []Object, os2 []Object) ObjectSliceDiff { + var diff ObjectSliceDiff + for _, o1 := range os1 { + found := false + for _, o2 := range os2 { + if o1.Path == o2.Path { + found = true + if o1.LastModified != o2.LastModified { + diff.Updated = append(diff.Updated, o2) + } + break + } + } + if !found { + diff.Removed = append(diff.Removed, o1) + } + } + for _, o2 := range os2 { + found := false + for _, o1 := range os1 { + if o2.Path == o1.Path { + found = true + break + } + } + if !found { + diff.Added = append(diff.Added, o2) + } + } + diff.Change = len(diff.Removed)+len(diff.Added)+len(diff.Updated) > 0 + return diff +} + +func cleanPrefix(prefix string) string { + return strings.Trim(prefix, "/") +} + +func removePrefixFromObjectPath(prefix string, path string) string { + if prefix == "" { + return path + } + path = strings.Replace(path, fmt.Sprintf("%s/", prefix), "", 1) + return path +} + +func objectPathIsInvalid(path string) bool { + return strings.Contains(path, "/") || path == "" +} diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go new file mode 100644 index 0000000..3772ffc --- /dev/null +++ b/pkg/storage/storage_test.go @@ -0,0 +1,176 @@ +package storage + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type StorageTestSuite struct { + suite.Suite + StorageBackends map[string]Backend + TempDirectory string +} + +func (suite *StorageTestSuite) setupStorageBackends() { + timestamp := time.Now().Format("20060102150405") + suite.TempDirectory = fmt.Sprintf("../../.test/storage-storage/%s", timestamp) + suite.StorageBackends = make(map[string]Backend) + suite.StorageBackends["LocalFilesystem"] = Backend(NewLocalFilesystemBackend(suite.TempDirectory)) + + // create empty dir in local storage to make sure it doesnt end up in ListObjects + err := os.MkdirAll(fmt.Sprintf("%s/%s", suite.TempDirectory, "ignoreme"), 0777) + suite.Nil(err, "No error creating ignored dir in local storage") + + if os.Getenv("TEST_CLOUD_STORAGE") == "1" { + prefix := fmt.Sprintf("unittest/%s", timestamp) + s3Bucket := os.Getenv("TEST_STORAGE_AMAZON_BUCKET") + s3Region := os.Getenv("TEST_STORAGE_AMAZON_REGION") + gcsBucket := os.Getenv("TEST_STORAGE_GOOGLE_BUCKET") + suite.StorageBackends["AmazonS3"] = Backend(NewAmazonS3Backend(s3Bucket, prefix, s3Region)) + suite.StorageBackends["GoogleCS"] = Backend(NewGoogleCSBackend(gcsBucket, prefix)) + } +} + +func (suite *StorageTestSuite) SetupSuite() { + suite.setupStorageBackends() + + for i := 1; i <= 9; i++ { + data := []byte(fmt.Sprintf("test content %d", i)) + path := fmt.Sprintf("test%d.txt", i) + for key, backend := range suite.StorageBackends { + err := backend.PutObject(path, data) + message := fmt.Sprintf("no error putting object %s using %s backend", path, key) + suite.Nil(err, message) + } + } + + for key, backend := range suite.StorageBackends { + if key == "LocalFilesystem" { + continue + } + data := []byte("skipped object") + path := "this/is/a/skipped/object.txt" + err := backend.PutObject(path, data) + message := fmt.Sprintf("no error putting skipped object %s using %s backend", path, key) + suite.Nil(err, message) + } +} + +func (suite *StorageTestSuite) TearDownSuite() { + defer os.RemoveAll(suite.TempDirectory) + + for i := 1; i <= 9; i++ { + path := fmt.Sprintf("test%d.txt", i) + for key, backend := range suite.StorageBackends { + err := backend.DeleteObject(path) + message := fmt.Sprintf("no error deleting object %s using %s backend", path, key) + suite.Nil(err, message) + } + } + + for key, backend := range suite.StorageBackends { + if key == "LocalFilesystem" { + continue + } + path := "this/is/a/skipped/object.txt" + err := backend.DeleteObject(path) + message := fmt.Sprintf("no error deleting skipped object %s using %s backend", path, key) + suite.Nil(err, message) + } +} + +func (suite *StorageTestSuite) TestListObjects() { + for key, backend := range suite.StorageBackends { + objects, err := backend.ListObjects() + message := fmt.Sprintf("no error listing objects using %s backend", key) + suite.Nil(err, message) + expectedNumObjects := 9 + message = fmt.Sprintf("%d objects listed using %s backend", expectedNumObjects, key) + suite.Equal(expectedNumObjects, len(objects), message) + for i, object := range objects { + path := fmt.Sprintf("test%d.txt", (i + 1)) + message = fmt.Sprintf("object %s found in list objects using %s backend", path, key) + suite.Equal(path, object.Path, message) + } + } +} + +func (suite *StorageTestSuite) TestGetObject() { + for key, backend := range suite.StorageBackends { + for i := 1; i <= 9; i++ { + path := fmt.Sprintf("test%d.txt", i) + object, err := backend.GetObject(path) + message := fmt.Sprintf("no error getting object %s using %s backend", path, key) + suite.Nil(err, message) + message = fmt.Sprintf("object %s content as expected using %s backend", path, key) + suite.Equal(object.Content, []byte(fmt.Sprintf("test content %d", i)), message) + } + } +} + +func (suite *StorageTestSuite) TestHasSuffix() { + now := time.Now() + o1 := Object{ + Path: "mychart-0.1.0.tgz", + Content: []byte{}, + LastModified: now, + } + suite.True(o1.HasExtension("tgz"), "object has tgz suffix") + o2 := Object{ + Path: "mychart-0.1.0.txt", + Content: []byte{}, + LastModified: now, + } + suite.False(o2.HasExtension("tgz"), "object does not have tgz suffix") +} + +func (suite *StorageTestSuite) TestGetObjectSliceDiff() { + now := time.Now() + os1 := []Object{ + { + Path: "test1.txt", + Content: []byte{}, + LastModified: now, + }, + } + os2 := []Object{} + diff := GetObjectSliceDiff(os1, os2) + suite.True(diff.Change, "change detected") + suite.Equal(diff.Removed, os1, "removed slice populated") + suite.Empty(diff.Added, "added slice empty") + suite.Empty(diff.Updated, "updated slice empty") + + os2 = append(os2, os1[0]) + diff = GetObjectSliceDiff(os1, os2) + suite.False(diff.Change, "no change detected") + suite.Empty(diff.Removed, "removed slice empty") + suite.Empty(diff.Added, "added slice empty") + suite.Empty(diff.Updated, "updated slice empty") + + os2[0].LastModified = now.Add(1) + diff = GetObjectSliceDiff(os1, os2) + suite.True(diff.Change, "change detected") + suite.Empty(diff.Removed, "removed slice empty") + suite.Empty(diff.Added, "added slice empty") + suite.Equal(diff.Updated, os2, "updated slice populated") + + os2[0].LastModified = now + os2 = append(os2, Object{ + Path: "test2.txt", + Content: []byte{}, + LastModified: now, + }) + diff = GetObjectSliceDiff(os1, os2) + suite.True(diff.Change, "change detected") + suite.Empty(diff.Removed, "removed slice empty") + suite.Equal(diff.Added, []Object{os2[1]}, "added slice empty") + suite.Empty(diff.Updated, "updated slice empty") +} + +func TestStorageTestSuite(t *testing.T) { + suite.Run(t, new(StorageTestSuite)) +} diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh new file mode 100755 index 0000000..4596d12 --- /dev/null +++ b/scripts/acceptance.sh @@ -0,0 +1,25 @@ +#!/bin/bash -ex + +PY_REQUIRES="requests==2.18.4 robotframework==3.0.2" + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +if [ "$(uname)" == "Darwin" ]; then + PLATFORM="darwin" +else + PLATFORM="linux" +fi + +export PATH="$PWD/testbin:$PWD/bin/$PLATFORM/amd64:$PATH" + +export HELM_HOME="$PWD/.helm" +helm init --client-only + +if [ ! -d .venv/ ]; then + virtualenv -p $(which python2.7) .venv/ + .venv/bin/python .venv/bin/pip install $PY_REQUIRES +fi + +mkdir -p .robot/ +.venv/bin/robot --outputdir=.robot/ acceptance_tests/ diff --git a/scripts/mirror_k8s_repos.sh b/scripts/mirror_k8s_repos.sh new file mode 100755 index 0000000..178741c --- /dev/null +++ b/scripts/mirror_k8s_repos.sh @@ -0,0 +1,28 @@ +#!/bin/bash -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +trap "rm -f index.yaml" EXIT +mkdir -p mirror/ + +get_all_tgzs() { + local repo_url="$1" + rm -f index.yaml + wget $repo_url/index.yaml + tgzs="$(ruby -ryaml -e \ + "YAML.load_file('index.yaml')['entries'].each do |k,e|;for c in e;puts c['urls'][0];end;end")" + pushd mirror/ + for tgz in $tgzs; do + if [ ! -f "${tgz##*/}" ]; then + wget $tgz + fi + done + popd +} + +# Stable +get_all_tgzs https://kubernetes-charts.storage.googleapis.com + +# Incubator +get_all_tgzs https://kubernetes-charts-incubator.storage.googleapis.com diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..b05ae49 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,76 @@ +#!/bin/bash -ex + +VERSION="$1" +DOCKER_REPO="chartmuseum/chartmuseum" +REQUIRED_RELEASE_ENV_VARS=( + "RELEASE_AMAZON_BUCKET" + "RELEASE_AMAZON_REGION" +) + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +COMMIT="$(git rev-parse HEAD)" + +main() { + check_args + check_env_vars + docker_build + release_latest + release_stable +} + +check_args() { + if [ "$VERSION" == "" ]; then + echo "usage: release.sh " + fi +} + +check_env_vars() { + set +x + ALL_ENV_VARS_PRESENT="1" + for VAR in ${REQUIRED_RELEASE_ENV_VARS[@]}; do + if [ "${!VAR}" == "" ]; then + echo "missing required test env var: $VAR" + ALL_ENV_VARS_PRESENT="0" + fi + done + if [ "$ALL_ENV_VARS_PRESENT" == "0" ]; then + exit 1 + fi + set -x +} + +docker_build() { + docker build -t $DOCKER_REPO:latest . +} + +release_latest() { + echo "$COMMIT" > .latest.txt + aws s3 --region=$RELEASE_AMAZON_REGION cp --recursive bin/ \ + s3://$RELEASE_AMAZON_BUCKET/release/latest/bin/ + aws s3 --region=$RELEASE_AMAZON_REGION cp .latest.txt \ + s3://$RELEASE_AMAZON_BUCKET/release/latest.txt + docker push $DOCKER_REPO:latest +} + +release_stable() { + set +e + aws s3 --region=$RELEASE_AMAZON_REGION ls s3://$RELEASE_AMAZON_BUCKET/release/ \ + | grep -F "v${VERSION}/" + local rc="$?" + set -e + if [ "$rc" == "0" ]; then + echo "v${VERSION} has already been released. Skipping." + else + echo "v${VERSION}" > .stable.txt + aws s3 --region=$RELEASE_AMAZON_REGION cp --recursive bin/ \ + s3://$RELEASE_AMAZON_BUCKET/release/v${VERSION}/bin/ + aws s3 --region=$RELEASE_AMAZON_REGION cp .stable.txt \ + s3://$RELEASE_AMAZON_BUCKET/release/stable.txt + docker tag $DOCKER_REPO:latest $DOCKER_REPO:v${VERSION} + docker push $DOCKER_REPO:v${VERSION} + fi +} + +main diff --git a/scripts/setup_test_environment.sh b/scripts/setup_test_environment.sh new file mode 100755 index 0000000..0a3612f --- /dev/null +++ b/scripts/setup_test_environment.sh @@ -0,0 +1,65 @@ +#!/bin/bash -ex + +HELM_VERSION="2.6.1" +REQUIRED_TEST_ENV_VARS=( + "TEST_STORAGE_AMAZON_BUCKET" + "TEST_STORAGE_AMAZON_REGION" + "TEST_STORAGE_GOOGLE_BUCKET" +) + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +export PATH="$PWD/testbin:$PATH" +export HELM_HOME="$PWD/.helm" + +main() { + check_env_vars + install_helm + package_test_charts +} + +check_env_vars() { + set +x + ALL_ENV_VARS_PRESENT="1" + for VAR in ${REQUIRED_TEST_ENV_VARS[@]}; do + if [ "${!VAR}" == "" ]; then + echo "missing required test env var: $VAR" + ALL_ENV_VARS_PRESENT="0" + fi + done + if [ "$ALL_ENV_VARS_PRESENT" == "0" ]; then + exit 1 + fi + set -x +} + +install_helm() { + if [ ! -f "testbin/helm" ]; then + mkdir -p testbin/ + [ "$(uname)" == "Darwin" ] && PLATFORM="darwin" || PLATFORM="linux" + TARBALL="helm-v${HELM_VERSION}-${PLATFORM}-amd64.tar.gz" + wget "https://storage.googleapis.com/kubernetes-helm/${TARBALL}" + tar -C testbin/ -xzf $TARBALL + rm -f $TARBALL + pushd testbin/ + UNCOMPRESSED_DIR="$(find . -mindepth 1 -maxdepth 1 -type d)" + mv $UNCOMPRESSED_DIR/helm . + rm -rf $UNCOMPRESSED_DIR + chmod +x ./helm + popd + helm init --client-only + fi +} + +package_test_charts() { + pushd testdata/charts/ + for d in $(find . -maxdepth 1 -mindepth 1 -type d); do + pushd $d + helm package --sign --key helm-test --keyring ../../pgp/helm-test-key.secret . + popd + done + popd +} + +main diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..b3685e9 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,18 @@ +#!/bin/bash -ex + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/../ + +rm -rf .cover/ .test/ +mkdir .cover/ .test/ +trap "rm -rf .test/" EXIT + +for pkg in `go list ./... | grep -v /vendor/`; do + go test -v -covermode=atomic \ + -coverprofile=".cover/$(echo $pkg | sed 's/\//_/g').cover.out" $pkg +done + +echo "mode: set" > .cover/cover.out && cat .cover/*.cover.out | grep -v mode: | sort -r | \ + awk '{if($1 != last) {print $0;last=$1}}' >> .cover/cover.out + +go tool cover -html=.cover/cover.out -o=.cover/coverage.html diff --git a/testdata/charts/mychart/Chart.yaml b/testdata/charts/mychart/Chart.yaml new file mode 100644 index 0000000..db73e76 --- /dev/null +++ b/testdata/charts/mychart/Chart.yaml @@ -0,0 +1,2 @@ +name: mychart +version: 0.1.0 \ No newline at end of file diff --git a/testdata/charts/mychart/templates/pod.yaml b/testdata/charts/mychart/templates/pod.yaml new file mode 100644 index 0000000..ad4aaaf --- /dev/null +++ b/testdata/charts/mychart/templates/pod.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: '{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}}' +spec: + containers: + - image: busybox + name: '{{ .Chart.Name }}' + command: ['/bin/sh', '-c', 'while true; do echo {{ .Release.Name }}; sleep 5; done'] \ No newline at end of file diff --git a/testdata/pgp/NOTE.txt b/testdata/pgp/NOTE.txt new file mode 100644 index 0000000..05d9ca2 --- /dev/null +++ b/testdata/pgp/NOTE.txt @@ -0,0 +1,2 @@ +These files were copied directly from github.com/kubernetes/helm repo +in the pkg/provenance/testdata directory \ No newline at end of file diff --git a/testdata/pgp/helm-test-key.pub b/testdata/pgp/helm-test-key.pub new file mode 100644 index 0000000..38714f2 Binary files /dev/null and b/testdata/pgp/helm-test-key.pub differ diff --git a/testdata/pgp/helm-test-key.secret b/testdata/pgp/helm-test-key.secret new file mode 100644 index 0000000..a966aef Binary files /dev/null and b/testdata/pgp/helm-test-key.secret differ