From d541760f60192ecd83ad68656e43fd342f6e0872 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sun, 8 Jul 2018 16:18:21 -0700 Subject: [PATCH] scripts/maintenance/*-aws: Drop --workspace-dir We haven't set $WORKSPACE since 1dea5c84 (tests: Remove unused smoke.sh + tfvars file, 2017-10-04, coreos/tectonic-installer#2036), so there's no longer a need for the --workspace-dir options. Users who care where the scratch files live can set $TMPDIR: $ man 7 environ | grep TMPDIR | sed 's/ */ /g' | cut -b -67 * TMPDIR influences the path prefix of names created by tempnam(3) I'm still calling readlink on the mktemp output in case $TMPDIR (or /tmp, if $TMPDIR is unset) is a symlink. I'm also fixing --config-file, --exclude-file, and --tag-file. Previously we were using: CONFIG_FILE="/tmp/config/$(basename "$config_file")" and similar. But inside the container, /tmp/config is coming from the $tmp_dir volume mount. And when --config-file was set, we weren't writing the referenced content into $tmp_dir. Now we always write the content into $tmp_dir, regardless of whether the content is user-supplied or the script's default. Also avoid some parallel-call races by avoiding a shared /tmp/config (or ${workspace}/config). With the old approach, the trap rm call for one invocation could blow away a /tmp/config used by another invocation. With this commit, we use mktemp to give a secure, unique $tmp_dir. And once we have that, we can hard-code the paths to the config, tag, and exclude files inside $tmp_dir. --- scripts/maintenance/clean-aws.sh | 30 ++++++---------- scripts/maintenance/tag-aws.sh | 34 +++++++------------ .../tag_clean_aws_grafiti_job.groovy | 1 - 3 files changed, 23 insertions(+), 42 deletions(-) mode change 100644 => 100755 tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy diff --git a/scripts/maintenance/clean-aws.sh b/scripts/maintenance/clean-aws.sh index 514a2ef618..e4b1f46164 100755 --- a/scripts/maintenance/clean-aws.sh +++ b/scripts/maintenance/clean-aws.sh @@ -31,9 +31,6 @@ Options: is replaced with either the following days' date or date-override. Only use if --tag-file is not used. - --workspace-dir (optional) Parent directory for a temporary directory. /tmp is - used by default. - --dry-run (optional) If set, grafiti will only do a dry run, i.e. not delete any resources. @@ -46,7 +43,6 @@ region= config_file= tag_file= date_override= -workspace= dry_run= while [ $# -gt 0 ]; do @@ -78,10 +74,6 @@ while [ $# -gt 0 ]; do date_override="${2:-}" shift ;; - --workspace-dir) - workspace="${2:-}" - shift - ;; --dry-run) dry_run="$1" ;; @@ -119,19 +111,19 @@ fi set -e -tmp_dir="/tmp/config" -if [ -n "$workspace" ]; then - tmp_dir="$(readlink -m "${workspace}/config")" -fi +tmp_dir="$(readlink -m "$(mktemp -d clean-aws-XXXXXXXXXX)")" mkdir -p "$tmp_dir" trap 'rm -rf "$tmp_dir"; exit' EXIT -if [ -z "$config_file" ]; then - config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)" - echo "maxNumRequestRetries = 11" > "$config_file" +if [ -n "$config_file" ]; then + cat "$config_file" >"$tmp_dir/config.toml" +else + echo "maxNumRequestRetries = 11" >"$tmp_dir/config.toml" fi -if [ -z "$tag_file" ]; then +if [ -n "$tag_file" ]; then + cat "$tag_file" >"$tmp_dir/tag.json" +else tag_file="$(mktemp -p "$tmp_dir")" date_string="$(date "+%Y-%m-%d" -d "-1 day")\",\"$(date "+%Y-%-m-%-d" -d "-1 day")\", @@ -142,7 +134,7 @@ if [ -z "$tag_file" ]; then date_string="$date_override" fi - cat < "$tag_file" + cat <"$tmp_dir/tag.json" {"TagFilters":[{"Key":"expirationDate","Values":["${date_string}"]}]} EOF fi @@ -170,8 +162,8 @@ docker run -t --rm --name grafiti-deleter \ -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ -e AWS_REGION="$region" \ - -e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \ - -e TAG_FILE="/tmp/config/$(basename "$tag_file")" \ + -e CONFIG_FILE="/tmp/config/config.toml" \ + -e TAG_FILE="/tmp/config/tag.json" \ quay.io/coreos/grafiti:"${version}" \ bash -c "grafiti $dry_run --config \"\$CONFIG_FILE\" --ignore-errors delete --all-deps --delete-file \"\$TAG_FILE\"" diff --git a/scripts/maintenance/tag-aws.sh b/scripts/maintenance/tag-aws.sh index 7a0eb4054f..885a6703aa 100755 --- a/scripts/maintenance/tag-aws.sh +++ b/scripts/maintenance/tag-aws.sh @@ -35,9 +35,6 @@ Options: with 'expirationDate: some-date-string', where some-date-string is replaced with either the following days' date or date-override. - --workspace-dir (optional) Parent directory for a temporary directory. /tmp is - used by default. - --dry-run (optional) If set, grafiti will only do a dry run, i.e. not tag any resources. @@ -50,7 +47,6 @@ region= config_file= exclude_file= date_override= -workspace= start_hour=8 end_hour=1 dry_run= @@ -92,10 +88,6 @@ while [ $# -gt 0 ]; do date_override="${2:-}" shift ;; - --workspace-dir) - workspace="${2:-}" - shift - ;; --dry-run) dry_run="$1" ;; @@ -136,11 +128,7 @@ set -e # Tag all resources present in CloudTrail over the specified time period with the # following day's date as default, or with the DATE_VALUE_OVERRIDE value. # Format YYYY-MM-DD. -tmp_dir="/tmp/config" -if [ -n "$workspace" ]; then - tmp_dir="$(readlink -m "${workspace}/config")" -fi -mkdir -p "$tmp_dir" +tmp_dir="$(readlink -m "$(mktemp -d tag-aws-XXXXXXXXXX)")" trap 'rm -rf "$tmp_dir"; exit' EXIT date_string='now|strftime(\"%Y-%m-%d\")' @@ -150,9 +138,10 @@ fi # Configure grafiti to tag all resources created between START_HOUR and END_HOUR's # ago -if [ -z "$config_file" ]; then - config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)" - cat < "$config_file" +if [ -n "$config_file" ]; then + cat "$config_file" >"$tmp_dir/config.toml" +else + cat <"$tmp_dir/config.toml" endHour = -${end_hour} startHour = -${start_hour} includeEvent = false @@ -164,13 +153,14 @@ fi # Exclusion file prevents tagging of resources that already have tags with the key # "expirationDate" -if [ -z "$exclude_file" ]; then - exclude_file="$(mktemp -p "$tmp_dir")" - echo '{"TagFilters":[{"Key":"expirationDate","Values":[]}]}' > "$exclude_file" +if [ -n "$exclude_file" ]; then + cat "$exclude_file" >"$tmp_dir/exclude" +else + echo '{"TagFilters":[{"Key":"expirationDate","Values":[]}]}' >"$tmp_dir/exclude" fi echo "Tagging resources with the following configuration:" -cat "$config_file" +cat "$tmp_dir/config.toml" if [ -n "$dry_run" ]; then echo "Dry run flag set. Not tagging any resources." @@ -192,8 +182,8 @@ docker run -t --rm --name grafiti-tagger \ -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ -e AWS_REGION="$region" \ - -e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \ - -e TAG_FILE="/tmp/config/$(basename "$exclude_file")" \ + -e CONFIG_FILE="/tmp/config/config.toml" \ + -e TAG_FILE="/tmp/config/exclude" \ quay.io/coreos/grafiti:"${version}" \ bash -c "grafiti --config \"\$CONFIG_FILE\" parse | \ grafiti --config \"\$CONFIG_FILE\" filter --ignore-file \"\$TAG_FILE\" | \ diff --git a/tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy b/tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy old mode 100644 new mode 100755 index 6c786791eb..fe147a1381 --- a/tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy +++ b/tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy @@ -77,7 +77,6 @@ for region in "\${regions[@]}"; do \$SCRIPT_DIR/maintenance/\$TAG_CLEAN.sh \\ --grafiti-version "\$GRAFITI_VERSION" \\ --aws-region "\$region" \\ - --workspace-dir "\$WORKSPACE" \\ --force \\ \$DATE_OVERRIDE_FLAG done