mirror of
https://github.com/openshift/installer.git
synced 2026-02-05 15:47:14 +01:00
installer/scripts: AWS tag and delete scripts
scripts/maintenance: scripts that tag and delete AWS resources using the `grafiti` Docker container (see https://github.com/coreos/grafiti), and a script that only tags Route53 hosted zones using the AWS CLI.
This commit is contained in:
Eric Stroczynski
@@ -1,46 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# TODO: Use jq instead of sed everywhere
|
||||
# TODO: Make xargs ignore all errors (might be pre-empted): use xargs -I{} sh -c 'aws ec2 ... {} || true'
|
||||
# TODO: Delete all other kind of resources
|
||||
|
||||
echo Deleting all private zones
|
||||
aws route53 list-hosted-zones | jq ".HostedZones[] | select(.Config.PrivateZone == true) | .Id" | xargs -L 1 aws route53 delete-hosted-zone --id
|
||||
aws route53 list-resource-record-sets --hosted-zone-id
|
||||
|
||||
|
||||
echo Deleting all ASGs
|
||||
aws autoscaling describe-auto-scaling-groups | jq ".AutoScalingGroups[] .AutoScalingGroupName" | xargs -L 1 aws autoscaling delete-auto-scaling-group --force-delete --auto-scaling-group-name
|
||||
|
||||
echo Deleting all ELBs
|
||||
aws elb describe-load-balancers | jq ".LoadBalancerDescriptions[] .LoadBalancerName" | xargs -L 1 aws elb delete-load-balancer --load-balancer-name
|
||||
|
||||
echo Deleting all NAT gateways
|
||||
aws ec2 describe-nat-gateways | jq '.NatGateways[] | select(.State == "available") | .NatGatewayId' | xargs -L 1 -I {} aws ec2 delete-nat-gateway --nat-gateway-id {} 1> /dev/null
|
||||
|
||||
echo "Waiting for the Elastic Network Interfaces from the ELBs, and for the NAT gateways, to be detached/deleted"
|
||||
sleep 30
|
||||
|
||||
echo Disassociating/Releasing all EIPs
|
||||
aws ec2 describe-addresses | grep AssociationId | sed -E 's/^.*(eipassoc-[a-z0-9]+).*$/\1/' | xargs -L 1 aws ec2 disassociate-address --association-id
|
||||
aws ec2 describe-addresses | grep AllocationId | sed -E 's/^.*(eipalloc-[a-z0-9]+).*$/\1/' | xargs -L 1 aws ec2 release-address --allocation-id
|
||||
|
||||
echo Detaching/Deleting all Internet Gateways
|
||||
aws ec2 describe-internet-gateways | jq '.InternetGateways[] | "--internet-gateway-id=" + .InternetGatewayId + " --vpc-id=" + .Attachments?[].VpcId' -r | xargs -L 1 aws ec2 detach-internet-gateway
|
||||
aws ec2 describe-internet-gateways | jq '.InternetGateways[] .InternetGatewayId' | xargs -L 1 aws ec2 delete-internet-gateway --internet-gateway-id
|
||||
|
||||
echo Deleting all subnets
|
||||
aws ec2 describe-subnets | jq ".Subnets[] .SubnetId" | xargs -L 1 aws ec2 delete-subnet --subnet-id
|
||||
|
||||
echo "Deleting all route tables (except the main ones)"
|
||||
#aws ec2 describe-route-tables | jq '.RouteTables[] | select(.Routes?[] | select(.DestinationCidrBlock == "0.0.0.0/0")) | .RouteTableId' | xargs -L 1 aws ec2 delete-route --destination-cidr-block 0.0.0.0/0 --route-table-id # First, delete the default route, as they might be pointed at a non-existing gateway.
|
||||
# shellcheck disable=SC2016
|
||||
aws ec2 describe-route-tables --query 'RouteTables[?Associations[0].Main != `true`]' | jq ".[] .RouteTableId" | xargs -L 1 aws ec2 delete-route-table --route-table-id
|
||||
|
||||
echo Deleting security groups
|
||||
aws ec2 describe-security-groups | jq ".SecurityGroups[] | select(.GroupName != \"default\" and (.IpPermissions|length > 0)) | \"--group-id \" + .GroupId + \" --ip-permissions '\" + (.IpPermissions|tostring) + \"'\"" -r | xargs -L 1 aws ec2 revoke-security-group-ingress
|
||||
aws ec2 describe-security-groups | jq ".SecurityGroups[] | select(.GroupName != \"default\" and (.IpPermissionsEgress|length > 0)) | \"--group-id \" + .GroupId + \" --ip-permissions '\" + (.IpPermissionsEgress|tostring) + \"'\"" -r | xargs -L 1 aws ec2 revoke-security-group-egress
|
||||
aws ec2 describe-security-groups | jq '.SecurityGroups[] | select(.GroupName != "default") | .GroupId' | xargs -L 1 -I {} sh -c 'aws ec2 delete-security-group --group-id {} || true'
|
||||
|
||||
echo Deleting VPCs
|
||||
aws ec2 describe-vpcs | jq ".Vpcs[] .VpcId" | xargs -L 1 -I {} sh -c 'aws ec2 delete-vpc --vpc-id {} || true'
|
||||
168
installer/scripts/maintenance/clean-aws.sh
Executable file
168
installer/scripts/maintenance/clean-aws.sh
Executable file
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
|
||||
$(basename "$0") deletes AWS resources tagged with tags specified in a tag file.
|
||||
|
||||
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental variables must be set.
|
||||
|
||||
Options:
|
||||
|
||||
--force Override user input prompts. Useful for automation.
|
||||
|
||||
--grafiti-version Either the semver release version, ex. v0.1.1, or sha commit
|
||||
hash of a grafiti image hosted in quay.io.
|
||||
|
||||
--aws-region The AWS region you wish to query for taggable resources. This
|
||||
flag is optional if AWS_REGION is set. AWS_REGION overrides
|
||||
values passed in by this flag.
|
||||
|
||||
--config-file A grafiti configuration file. See an example at
|
||||
https://github.com/coreos/grafiti/blob/master/config.toml.
|
||||
|
||||
--tag-file A file containing a TagFilter list. See the AWS Resource Group
|
||||
Tagging API 'TagFilter' documentation for file structure.
|
||||
|
||||
--date-override (optional) Date of the format YYYY-MM-DD that overrides the
|
||||
default tag value of today's date. This script tags resources
|
||||
with 'expirationDate: some-date-string', where some-date-string
|
||||
is replaced with either the following days' date or date-override.
|
||||
Only use if --tag-file is not used.
|
||||
|
||||
--workspace-dir (optional) Parent directory for a temporary directory. /tmp is
|
||||
used by default.
|
||||
|
||||
--dry-run (optional) If set, grafiti will only do a dry run, i.e. not
|
||||
delete any resources.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
force=
|
||||
version=
|
||||
region=
|
||||
config_file=
|
||||
tag_file=
|
||||
date_override=
|
||||
workspace=
|
||||
dry_run=
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--help)
|
||||
usage
|
||||
exit
|
||||
;;
|
||||
--force)
|
||||
force=true
|
||||
;;
|
||||
--grafiti-version)
|
||||
version="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--aws-region)
|
||||
region="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--config-file)
|
||||
config_file="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--tag-file)
|
||||
tag_file="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--date-override)
|
||||
date_override="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--workspace-dir)
|
||||
workspace="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
dry_run="$1"
|
||||
;;
|
||||
*)
|
||||
echo "Flag '$2' is not supported."
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -n "$AWS_REGION" ]; then
|
||||
region="${AWS_REGION:-}"
|
||||
fi
|
||||
|
||||
if [ -z "$version" ]; then
|
||||
echo "Grafiti image version required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$region" ]; then
|
||||
echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$tag_file" ] && [ -n "$date_override" ]; then
|
||||
echo "Cannot use both --tag-file and --date-override flags simultaneously."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
tmp_dir="/tmp/config"
|
||||
if [ -n "$workspace" ]; then
|
||||
tmp_dir="$(readlink -m "${workspace}/config")"
|
||||
fi
|
||||
mkdir -p "$tmp_dir"
|
||||
trap 'rm -rf "$tmp_dir"; exit' EXIT
|
||||
|
||||
if [ -z "$config_file" ]; then
|
||||
config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)"
|
||||
echo "maxNumRequestRetries = 11" > "$config_file"
|
||||
fi
|
||||
|
||||
if [ -z "$tag_file" ]; then
|
||||
tag_file="$(mktemp -p "$tmp_dir")"
|
||||
|
||||
date_string="$(date "+%Y-%m-%d" -d "-1 day")\",\"$(date "+%Y-%-m-%-d" -d "-1 day")\",\"$(date +%Y-%m-%d)\",\"$(date +%Y-%-m-%-d)"
|
||||
if [ -n "$date_override" ]; then
|
||||
date_string="$date_override"
|
||||
fi
|
||||
|
||||
cat <<EOF > "$tag_file"
|
||||
{"TagFilters":[{"Key":"expirationDate","Values":["${date_string}"]}]}
|
||||
EOF
|
||||
fi
|
||||
|
||||
echo "Deleting resources with the following tags:"
|
||||
jq '.' "$tag_file"
|
||||
|
||||
if [ -n "$dry_run" ]; then
|
||||
echo "Dry run flag set. Not deleting any resources."
|
||||
fi
|
||||
|
||||
if [ ! $force ]; then
|
||||
read -rp "Proceed deleting these resources? [y/N]: " yn
|
||||
if [ "$yn" != "y" ]; then
|
||||
echo "Aborting deletion and cleaning up."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
trap 'docker stop grafiti-deleter && docker rm grafiti-deleter; exit' EXIT
|
||||
|
||||
docker run -t --rm --name grafiti-deleter \
|
||||
-v "$tmp_dir":/tmp/config:z \
|
||||
-e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
|
||||
-e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
|
||||
-e AWS_REGION="$region" \
|
||||
-e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \
|
||||
-e TAG_FILE="/tmp/config/$(basename "$tag_file")" \
|
||||
quay.io/coreos/grafiti:"${version}" \
|
||||
ash -c "grafiti $dry_run --config \"\$CONFIG_FILE\" --ignore-errors delete --all-deps --delete-file \"\$TAG_FILE\""
|
||||
|
||||
set +e
|
||||
196
installer/scripts/maintenance/tag-aws.sh
Executable file
196
installer/scripts/maintenance/tag-aws.sh
Executable file
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
|
||||
$(basename "$0") tags AWS resources with 'expirationDate: some-date-string',
|
||||
defaulting to the following days' date, and excludes all resources tagged with
|
||||
tag keys/values specified in an 'exclude' file. Requires that both 'jq' and the
|
||||
AWS CLI are installed.
|
||||
|
||||
AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental variables must be set.
|
||||
|
||||
Options:
|
||||
|
||||
--force Override user input prompts. Useful for automation.
|
||||
|
||||
--grafiti-version Either the semver release version, ex. v0.1.1, or sha commit
|
||||
hash of a grafiti image hosted in quay.io.
|
||||
|
||||
--aws-region The AWS region you wish to query for taggable resources. This
|
||||
flag is optional if AWS_REGION is set. AWS_REGION overrides
|
||||
values passed in by this flag.
|
||||
|
||||
--config-file A grafiti configuration file. See an example at
|
||||
https://github.com/coreos/grafiti/blob/master/config.toml.
|
||||
|
||||
--exclude-file A file containing a JSON array of Key/Value pair objects.
|
||||
|
||||
--start-hour Integer hour to start looking at CloudTrail logs. Defaults to 8.
|
||||
|
||||
--end-hour Integer hour to end looking at CloudTrail logs. Defaults to 1.
|
||||
|
||||
--date-override (optional) Date of the format YYYY-MM-DD that overrides the
|
||||
default tag value of today's date. This script tags resources
|
||||
with 'expirationDate: some-date-string', where some-date-string
|
||||
is replaced with either the following days' date or date-override.
|
||||
|
||||
--workspace-dir (optional) Parent directory for a temporary directory. /tmp is
|
||||
used by default.
|
||||
|
||||
--dry-run (optional) If set, grafiti will only do a dry run, i.e. not tag
|
||||
any resources.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
force=
|
||||
version=
|
||||
region=
|
||||
config_file=
|
||||
exclude_file=
|
||||
date_override=
|
||||
workspace=
|
||||
start_hour=8
|
||||
end_hour=1
|
||||
dry_run=
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--help)
|
||||
usage
|
||||
exit
|
||||
;;
|
||||
--force)
|
||||
force=true
|
||||
;;
|
||||
--grafiti-version)
|
||||
version="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--aws-region)
|
||||
region="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--config-file)
|
||||
config_file="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--exclude-file)
|
||||
exclude_file="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--start-hour)
|
||||
start_hour="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--end-hour)
|
||||
end_hour="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--date-override)
|
||||
date_override="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--workspace-dir)
|
||||
workspace="${2:-}"
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
dry_run="$1"
|
||||
;;
|
||||
*)
|
||||
echo "Flag '$2' is not supported."
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -n "$AWS_REGION" ]; then
|
||||
region="${AWS_REGION:-}"
|
||||
fi
|
||||
|
||||
if [ -z "$version" ]; then
|
||||
echo "Grafiti image version required."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$start_hour" ] || [ -z "$end_hour" ]; then
|
||||
echo "Start hour and end hour must be specified."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$region" ]; then
|
||||
echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Tag all resources present in CloudTrail over the specified time period with the
|
||||
# following day's date as default, or with the DATE_VALUE_OVERRIDE value.
|
||||
# Format YYYY-MM-DD.
|
||||
tmp_dir="/tmp/config"
|
||||
if [ -n "$workspace" ]; then
|
||||
tmp_dir="$(readlink -m "${workspace}/config")"
|
||||
fi
|
||||
mkdir -p "$tmp_dir"
|
||||
trap 'rm -rf "$tmp_dir"; exit' EXIT
|
||||
|
||||
date_string='now|strftime(\"%Y-%m-%d\")'
|
||||
if [ -n "$date_override" ]; then
|
||||
date_string='\"'"${date_override}"'\"'
|
||||
fi
|
||||
|
||||
# Configure grafiti to tag all resources created between START_HOUR and END_HOUR's
|
||||
# ago
|
||||
if [ -z "$config_file" ]; then
|
||||
config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)"
|
||||
cat <<EOF > "$config_file"
|
||||
endHour = -${end_hour}
|
||||
startHour = -${start_hour}
|
||||
includeEvent = false
|
||||
tagPatterns = [
|
||||
"{expirationDate: ${date_string}}"
|
||||
]
|
||||
EOF
|
||||
fi
|
||||
|
||||
# Exclusion file prevents tagging of resources that already have tags with the key
|
||||
# "expirationDate"
|
||||
if [ -z "$exclude_file" ]; then
|
||||
exclude_file="$(mktemp -p "$tmp_dir")"
|
||||
echo '{"TagFilters":[{"Key":"expirationDate","Values":[]}]}' > "$exclude_file"
|
||||
fi
|
||||
|
||||
echo "Tagging resources with the following configuration:"
|
||||
cat "$config_file"
|
||||
|
||||
if [ -n "$dry_run" ]; then
|
||||
echo "Dry run flag set. Not tagging any resources."
|
||||
fi
|
||||
|
||||
if [ ! $force ]; then
|
||||
read -rp "Proceed tagging these resources? [y/N]: " yn
|
||||
if [ "$yn" != "y" ]; then
|
||||
echo "Aborting tagging and cleaning up."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
trap 'docker stop grafiti-tagger && docker rm grafiti-tagger; exit' EXIT
|
||||
|
||||
docker run -t --rm --name grafiti-tagger \
|
||||
-v "$tmp_dir":/tmp/config:z \
|
||||
-e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
|
||||
-e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
|
||||
-e AWS_REGION="$region" \
|
||||
-e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \
|
||||
-e TAG_FILE="/tmp/config/$(basename "$exclude_file")" \
|
||||
quay.io/coreos/grafiti:"${version}" \
|
||||
bash -c "grafiti --config \"\$CONFIG_FILE\" parse | \
|
||||
grafiti --config \"\$CONFIG_FILE\" filter --ignore-file \"\$TAG_FILE\" | \
|
||||
grafiti $dry_run --config \"\$CONFIG_FILE\" tag"
|
||||
|
||||
set +e
|
||||
100
installer/scripts/maintenance/tag-route53-hosted-zones.sh
Executable file
100
installer/scripts/maintenance/tag-route53-hosted-zones.sh
Executable file
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
|
||||
$(basename "$0") tags AWS Route53 Hosted Zones with an 'expirationDate' of tomorrow.
|
||||
Requires that both 'jq' and the AWS CLI are installed.
|
||||
|
||||
Either the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental variables
|
||||
must be set, or ~/.aws/credentials must contain valid AWS credentials.
|
||||
|
||||
Options:
|
||||
|
||||
--force Override user input prompts. Useful for automation.
|
||||
|
||||
--date-override (optional) Date of the format YYYY-MM-DD that overrides the
|
||||
default tag value of today's date. This script tags resources
|
||||
with 'expirationDate: some-date-string', where some-date-string
|
||||
is replaced with either the following days' date or date-override.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
force=
|
||||
date_override=
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--help)
|
||||
usage
|
||||
exit
|
||||
;;
|
||||
--force)
|
||||
force=true
|
||||
;;
|
||||
--date-override)
|
||||
date_override="${2:-}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "Flag '$2' is not supported."
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if ! command -v jq > /dev/null || ! command -v aws > /dev/null; then
|
||||
"Dependencies not installed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Tag all Route53 hosted zones that do not already have a tag with the same keys,
|
||||
# in this case 'expirationDate', with today's date as default, or
|
||||
# with the DATE_VALUE_OVERRIDE value. Format YYYY-MM-DD.
|
||||
date_string="$(date "+%Y-%m-%d")"
|
||||
if [ -n "$date_override" ]; then
|
||||
date_string="${date_override}"
|
||||
fi
|
||||
|
||||
tags="[{\"Key\":\"expirationDate\",\"Value\":\"$date_string\"}]"
|
||||
|
||||
echo "Tagging hosted zones with the following tags:"
|
||||
echo "$tags"
|
||||
|
||||
if [ ! $force ]; then
|
||||
read -rp "Proceed tagging these resources? [y/N]: " yn
|
||||
if [ "$yn" != "y" ]; then
|
||||
echo "Aborting tagging and cleaning up."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
private_zones=$(aws route53 list-hosted-zones | \
|
||||
jq ".HostedZones[] | select(.Config.PrivateZone == true) | .Id" | \
|
||||
sed "s@\"@@g")
|
||||
|
||||
for key in $(echo -e "$tags" | jq ".[].Key"); do
|
||||
for zone in $private_zones; do
|
||||
zone="${zone##*/}"
|
||||
is_not_tagged=$(aws route53 list-tags-for-resource \
|
||||
--resource-type hostedzone \
|
||||
--resource-id "$zone" | \
|
||||
jq ".ResourceTagSet | select(.Tags[]? | .Key == $key) | .ResourceId")
|
||||
if [ -z "$is_not_tagged" ]; then
|
||||
if aws route53 change-tags-for-resource \
|
||||
--resource-type hostedzone \
|
||||
--add-tags "$(echo -e "$tags")" \
|
||||
--resource-id "${zone##*/}"; then
|
||||
echo "Tagged hosted zone ${zone##*/}"
|
||||
else
|
||||
echo "Error tagging hosted zone ${zone##*/}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
set +e
|
||||
Reference in New Issue
Block a user