From 184a0bfe360f8110bf20d859b83ad357bcf46c22 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 26 Jul 2018 09:35:07 -0700 Subject: [PATCH] scripts/maintenance: Fix dates (tomorrow vs. today, etc.) tag-aws.sh is using grafiti, whose tagPatterns takes jq expressions [1]. We've been using strftime since the script landed in 82bdd9fe (installer/scripts: AWS tag and delete scripts, 2017-06-28, coreos/tectonic-installer#1239). jq's strftime doesn't respect your configured $TZ, but the coming jq 1.6 will add strflocaltime which does [2,3]. jq uses seconds since the epoch for date-time values [4]. You can test the new construct with: $ jq --null-input --raw-output 'now + 24*60*60 | strftime("%Y-%m-%d")' 2018-07-27 -d is not part of the POSIX date specification [5], but it (and the 'tomorrow' value) are supported by GNU Coreutils [6,7]. We've been using -d in clean-aws.sh for a while now, so this is now a new dependency. I've also dropped date_override, since we can just set date_string directly. And I've shuffled around some of the conditionals to avoid calling the 'date' and 'jq' commands needlessly when --date-override is set. I've also replaced the multiple date calls in clean-aws.sh with a single call to jq. jq was already a required dependency for this script, and only needing a single child process is much faster: $ time for i in $(seq 100); do A="$(jq --null-input '[["%Y-%m-%d", "%Y-%-m-%-d", "%m-%d-%Y", "%m-%-d-%-Y", "%-m-%-d-%-Y", "%d-%m-%Y", "%d-%-m-%-Y"][] | . as $format | [now, now - 24*60*60][] | strftime($format)]')"; done real 0m0.256s user 0m0.186s sys 0m0.077s $ time for i in $(seq 100); do A="$(date "+%Y-%m-%d" -d "-1 day")\",\"$(date "+%Y-%-m-%-d" -d "-1 day")\",\"$(date "+%m-%-d-%-Y" -d "-1 day")\",\"$(date "+%-m-%-d-%-Y" -d "-1 day")\",\"$(date "+%d-%m-%-Y" -d "-1 day")\",\"$(date "+%d-%-m-%-Y" -d "-1 day")\",\"$(date +%m-%d-%Y)\",\"$(date +%d-%m-%Y)\",\"$(date +%d-%-m-%Y)\",\"$(date +%Y-%m-%d)\",\"$(date +%Y-%-m-%-d)"; done real 0m1.358s user 0m0.604s sys 0m0.832s And that's despite the fact that the old approach skipped some formats for today (e.g. %m-%-d-%-Y had been only used to format yesterday). The plethora of date formats are mostly from 39952635 (ci: add more date format when grafiti apply the cleanning, 2017-09-12, coreos/tectonic-installer#1890), although we've had some since 82bdd9fe. The motivation seems to be matching human-generated tags [8], which are less reliably formatted. [1]: https://github.com/coreos/grafiti/blob/89a8bc92ad7fde49cd3dd78197c6b3616a857f36/README.md#configure-grafiti [2]: https://github.com/stedolan/jq/wiki/FAQ [3]: https://github.com/stedolan/jq/commit/06f20603f6022b721fa6a001969fbbb1111f113d [4]: https://stedolan.github.io/jq/manual/v1.5/#Dates [5]: http://pubs.opengroup.org/onlinepubs/9699919799/utilities/date.html [6]: https://www.gnu.org/software/coreutils/manual/html_node/Options-for-date.html [7]: https://www.gnu.org/software/coreutils/manual/html_node/Relative-items-in-date-strings.html [8]: https://github.com/coreos/tectonic-installer/pull/1890#issuecomment-328904664 --- scripts/maintenance/clean-aws.sh | 25 ++++++++----------- scripts/maintenance/tag-aws.sh | 17 ++++++------- .../maintenance/tag-route53-hosted-zones.sh | 17 ++++++------- 3 files changed, 26 insertions(+), 33 deletions(-) diff --git a/scripts/maintenance/clean-aws.sh b/scripts/maintenance/clean-aws.sh index 2854fa10cf..2497d247e1 100755 --- a/scripts/maintenance/clean-aws.sh +++ b/scripts/maintenance/clean-aws.sh @@ -27,11 +27,10 @@ Options: --tag-file A file containing a TagFilter list. See the AWS Resource Group Tagging API 'TagFilter' documentation for file structure. - --date-override (optional) Date of the format YYYY-MM-DD that overrides the - default tag value of today's date. This script tags resources - with 'expirationDate: some-date-string', where some-date-string - is replaced with either the following days' date or date-override. - Only use if --tag-file is not used. + --date-override (optional) Date of the format YYYY-MM-DD to delete resources + tagged with 'expirationDate: some-date-string'. By default, + this script deletes resources which expired yesterday or + today. Not compatible with --tag-file. --dry-run (optional) If set, grafiti will only do a dry run, i.e. not delete any resources. @@ -44,7 +43,7 @@ version= region= config_file= tag_file= -date_override= +date_string= dry_run= while [ $# -gt 0 ]; do @@ -73,7 +72,7 @@ while [ $# -gt 0 ]; do shift ;; --date-override) - date_override="${2:-}" + date_string="[\"${2:-}\"]" shift ;; --dry-run) @@ -112,7 +111,7 @@ if [ -z "$version" ]; then exit 1 fi -if [ -n "$tag_file" ] && [ -n "$date_override" ]; then +if [ -n "$tag_file" ] && [ -n "$date_string" ]; then echo "Cannot use both --tag-file and --date-override flags simultaneously." >&2 exit 1 fi @@ -132,16 +131,12 @@ fi if [ -n "$tag_file" ]; then cat "$tag_file" >"$tmp_dir/tag.json" else - date_string="$(date "+%Y-%m-%d" -d "-1 day")\",\"$(date "+%Y-%-m-%-d" -d "-1 day")\", - \"$(date "+%m-%-d-%-Y" -d "-1 day")\",\"$(date "+%-m-%-d-%-Y" -d "-1 day")\",\"$(date "+%d-%m-%-Y" -d "-1 day")\", - \"$(date "+%d-%-m-%-Y" -d "-1 day")\",\"$(date +%m-%d-%Y)\",\"$(date +%d-%m-%Y)\", - \"$(date +%d-%-m-%Y)\",\"$(date +%Y-%m-%d)\",\"$(date +%Y-%-m-%-d)" - if [ -n "$date_override" ]; then - date_string="$date_override" + if [ -z "$date_string" ]; then + date_string="$(jq --null-input '[["%Y-%m-%d", "%Y-%-m-%-d", "%m-%d-%Y", "%m-%-d-%-Y", "%-m-%-d-%-Y", "%d-%m-%Y", "%d-%-m-%-Y"][] | . as $format | [now, now - 24*60*60][] | strftime($format)]')" fi cat <"$tmp_dir/tag.json" -{"TagFilters":[{"Key":"expirationDate","Values":["${date_string}"]}]} +{"TagFilters":[{"Key":"expirationDate","Values":${date_string}}]} EOF fi diff --git a/scripts/maintenance/tag-aws.sh b/scripts/maintenance/tag-aws.sh index 0404b6d53e..211356de0c 100755 --- a/scripts/maintenance/tag-aws.sh +++ b/scripts/maintenance/tag-aws.sh @@ -4,7 +4,7 @@ usage() { cat <