diff --git a/pkg/destroy/aws/aws.go b/pkg/destroy/aws/aws.go index 033a990fc3..f3f410c432 100644 --- a/pkg/destroy/aws/aws.go +++ b/pkg/destroy/aws/aws.go @@ -9,21 +9,24 @@ import ( awsv2 "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" configv2 "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" ec2v2 "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/efs" elb "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" iamv2 "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" + tagtypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" "github.com/aws/aws-sdk-go-v2/service/route53" route53types "github.com/aws/aws-sdk-go-v2/service/route53/types" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,6 +37,7 @@ import ( awssession "github.com/openshift/installer/pkg/asset/installconfig/aws" "github.com/openshift/installer/pkg/destroy/providers" "github.com/openshift/installer/pkg/types" + awstypes "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/version" ) @@ -67,6 +71,7 @@ type ClusterUninstaller struct { ClusterID string ClusterDomain string HostedZoneRole string + endpoints []awstypes.ServiceEndpoint // Session is the AWS session to be used for deletion. If nil, a // new session will be created based on the usual credential @@ -175,6 +180,7 @@ func New(logger logrus.FieldLogger, metadata *types.ClusterMetadata) (providers. ClusterDomain: metadata.AWS.ClusterDomain, Session: session, HostedZoneRole: metadata.AWS.HostedZoneRole, + endpoints: metadata.AWS.ServiceEndpoints, EC2Client: ec2Client, IAMClient: iamClient, ELBClient: elbclient, @@ -198,6 +204,25 @@ func (o *ClusterUninstaller) Run() (*types.ClusterQuota, error) { return nil, err } +func createResourceTaggingClientWithConfig(cfg awsv2.Config, region string, endpoints []awstypes.ServiceEndpoint) *resourcegroupstaggingapi.Client { + return resourcegroupstaggingapi.NewFromConfig(cfg, func(options *resourcegroupstaggingapi.Options) { + options.Region = region + for _, endpoint := range endpoints { + if strings.EqualFold(endpoint.Name, "resourcegroupstaggingapi") { + options.BaseEndpoint = awsv2.String(endpoint.URL) + } + } + }) +} + +func createResourceTaggingClient(region string, endpoints []awstypes.ServiceEndpoint) (*resourcegroupstaggingapi.Client, error) { + cfg, err := awssession.GetConfigWithOptions(context.Background(), configv2.WithRegion(region)) + if err != nil { + return nil, fmt.Errorf("failed to create AWS config for resource tagging client: %w", err) + } + return createResourceTaggingClientWithConfig(cfg, region, endpoints), nil +} + // RunWithContext runs the uninstall process with a context. // The first return is the list of ARNs for resources that could not be destroyed. func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, error) { @@ -219,16 +244,31 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro Fn: request.MakeAddToUserAgentHandler("OpenShift/4.x Destroyer", version.Raw), }) - tagClients := []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI{ - resourcegroupstaggingapi.New(awsSession), + baseTaggingClient, err := createResourceTaggingClient(o.Region, o.endpoints) + if err != nil { + return nil, err } + tagClients := []*resourcegroupstaggingapi.Client{baseTaggingClient} if o.HostedZoneRole != "" { - cfg := awssession.GetR53ClientCfg(awsSession, o.HostedZoneRole) + cfg, err := awssession.GetConfigWithOptions(ctx, configv2.WithRegion(endpointUSEast1)) + if err != nil { + return nil, fmt.Errorf("failed to create AWS config for resource tagging client: %w", err) + } + stsSvc, err := awssession.NewSTSClient(ctx, awssession.EndpointOptions{ + Region: endpointUSEast1, + Endpoints: o.endpoints, + }, sts.WithAPIOptions(middleware.AddUserAgentKeyValue("OpenShift/4.x Destroyer", version.Raw))) + if err != nil { + return nil, fmt.Errorf("failed to create STS client: %w", err) + } + + creds := stscreds.NewAssumeRoleProvider(stsSvc, o.HostedZoneRole) + cfg.Credentials = awsv2.NewCredentialsCache(creds) // This client is specifically for finding route53 zones, // so it needs to use the global us-east-1 region. - cfg.Region = aws.String(endpoints.UsEast1RegionID) - tagClients = append(tagClients, resourcegroupstaggingapi.New(awsSession, cfg)) + + tagClients = append(tagClients, createResourceTaggingClientWithConfig(cfg, endpointUSEast1, o.endpoints)) } switch o.Region { @@ -238,13 +278,19 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro break case endpoints.UsGovEast1RegionID, endpoints.UsGovWest1RegionID: if o.Region != endpoints.UsGovWest1RegionID { - tagClients = append(tagClients, - resourcegroupstaggingapi.New(awsSession, aws.NewConfig().WithRegion(endpoints.UsGovWest1RegionID))) + tagClient, err := createResourceTaggingClient(endpoints.UsGovWest1RegionID, o.endpoints) + if err != nil { + return nil, fmt.Errorf("failed to create resource tagging client for usgov-west-1: %w", err) + } + tagClients = append(tagClients, tagClient) } default: if o.Region != endpoints.UsEast1RegionID { - tagClients = append(tagClients, - resourcegroupstaggingapi.New(awsSession, aws.NewConfig().WithRegion(endpoints.UsEast1RegionID))) + tagClient, err := createResourceTaggingClientWithConfig(endpoints.UsEast1RegionID, o.endpoints) + if err != nil { + return nil, fmt.Errorf("failed to create resource tagging client for default us-east-1: %w", err) + } + tagClients = append(tagClients, tagClient) } } @@ -275,7 +321,7 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro // Terminate EC2 instances. The instances need to be terminated first so that we can ensure that there is nothing // running on the cluster creating new resources while we are attempting to delete resources, which could leak // the new resources. - err = o.DeleteEC2Instances(ctx, awsSession, resourcesToDelete, deleted, tracker) + err = o.DeleteEC2Instances(ctx, resourcesToDelete, deleted, tracker) if err != nil { return resourcesToDelete.UnsortedList(), err } @@ -284,7 +330,7 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro err = wait.PollImmediateUntil( time.Second*10, func() (done bool, err error) { - newlyDeleted, loopError := o.DeleteResources(ctx, awsSession, resourcesToDelete.UnsortedList(), tracker) + newlyDeleted, loopError := o.DeleteResources(ctx, resourcesToDelete.UnsortedList(), tracker) // Delete from the resources-to-delete set so that the current state of the resources to delete can be // returned if the context is completed. resourcesToDelete = resourcesToDelete.Difference(newlyDeleted) @@ -314,7 +360,7 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro return resourcesToDelete.UnsortedList(), err } - err = o.removeSharedTags(ctx, awsSession, tagClients, tracker) + err = o.removeSharedTags(ctx, tagClients, tracker) if err != nil { return nil, err } @@ -333,7 +379,7 @@ func (o *ClusterUninstaller) findUntaggableResources(ctx context.Context, delete profile := fmt.Sprintf("%s-%s-profile", o.ClusterID, profileType) response, err := o.IAMClient.GetInstanceProfile(ctx, &iamv2.GetInstanceProfileInput{InstanceProfileName: &profile}) if err != nil { - if strings.Contains(HandleErrorCode(err), "NoSuchEntity") { + if strings.Contains(handleErrorCode(err), "NoSuchEntity") { continue } return resources, fmt.Errorf("failed to get IAM instance profile: %w", err) @@ -352,11 +398,11 @@ func (o *ClusterUninstaller) findUntaggableResources(ctx context.Context, delete // deleted - the resources that have already been deleted. Any resources specified in this set will be ignored. func (o *ClusterUninstaller) findResourcesToDelete( ctx context.Context, - tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, + tagClients []*resourcegroupstaggingapi.Client, iamRoleSearch *IamRoleSearch, iamUserSearch *IamUserSearch, deleted sets.Set[string], -) (sets.Set[string], []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, error) { +) (sets.Set[string], []*resourcegroupstaggingapi.Client, error) { var errs []error resources, tagClients, err := FindTaggedResourcesToDelete(ctx, o.Logger, tagClients, o.Filters, iamRoleSearch, iamUserSearch, deleted) if err != nil { @@ -380,14 +426,14 @@ func (o *ClusterUninstaller) findResourcesToDelete( func FindTaggedResourcesToDelete( ctx context.Context, logger logrus.FieldLogger, - tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, + tagClients []*resourcegroupstaggingapi.Client, filters []Filter, iamRoleSearch *IamRoleSearch, iamUserSearch *IamUserSearch, deleted sets.Set[string], -) (sets.Set[string], []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, error) { +) (sets.Set[string], []*resourcegroupstaggingapi.Client, error) { resources := sets.New[string]() - var tagClientsWithResources []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI + var tagClientsWithResources []*resourcegroupstaggingapi.Client var errs []error // Find resources by tag @@ -402,7 +448,7 @@ func FindTaggedResourcesToDelete( if len(resourcesInTagClient) > 0 || err != nil { tagClientsWithResources = append(tagClientsWithResources, tagClient) } else { - logger.Debugf("no deletions from %s, removing client", *tagClient.Config.Region) + logger.Debugf("no deletions from %s, removing client", tagClient.Options().Region) } } @@ -434,39 +480,37 @@ func FindTaggedResourcesToDelete( func findResourcesByTag( ctx context.Context, logger logrus.FieldLogger, - tagClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI, + tagClient *resourcegroupstaggingapi.Client, filters []Filter, deleted sets.Set[string], ) (sets.Set[string], error) { resources := sets.New[string]() for _, filter := range filters { - logger.Debugf("search for matching resources by tag in %s matching %#+v", *tagClient.Config.Region, filter) - tagFilters := make([]*resourcegroupstaggingapi.TagFilter, 0, len(filter)) + logger.Debugf("search for matching resources by tag in %s matching %#+v", tagClient.Options().Region, filter) + tagFilters := make([]tagtypes.TagFilter, 0, len(filter)) for key, value := range filter { - tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ + tagFilters = append(tagFilters, tagtypes.TagFilter{ Key: aws.String(key), - Values: []*string{aws.String(value)}, + Values: []string{value}, }) } - err := tagClient.GetResourcesPagesWithContext( - ctx, - &resourcegroupstaggingapi.GetResourcesInput{TagFilters: tagFilters}, - func(results *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { - for _, resource := range results.ResourceTagMappingList { - arnString := *resource.ResourceARN - if !deleted.Has(arnString) { - resources.Insert(arnString) - } + + paginator := resourcegroupstaggingapi.NewGetResourcesPaginator(tagClient, &resourcegroupstaggingapi.GetResourcesInput{TagFilters: tagFilters}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return resources, fmt.Errorf("failed to fetch resources by tag: %w", err) + } + + for _, resource := range page.ResourceTagMappingList { + arnString := *resource.ResourceARN + if !deleted.Has(arnString) { + resources.Insert(arnString) } - return !lastPage - }, - ) - if err != nil { - err = errors.Wrap(err, "get tagged resources") - logger.Info(err) - return resources, err + } } } + return resources, nil } @@ -475,7 +519,7 @@ func findResourcesByTag( // resources - the resources to be deleted. // // The first return is the ARNs of the resources that were successfully deleted. -func (o *ClusterUninstaller) DeleteResources(ctx context.Context, awsSession *session.Session, resources []string, tracker *ErrorTracker) (sets.Set[string], error) { +func (o *ClusterUninstaller) DeleteResources(ctx context.Context, resources []string, tracker *ErrorTracker) (sets.Set[string], error) { deleted := sets.New[string]() for _, arnString := range resources { l := o.Logger.WithField("arn", arnString) @@ -638,7 +682,7 @@ func deleteRoute53(ctx context.Context, client *route53.Client, arn arn.ARN, log if err != nil { // In some cases AWS may return the zone in the list of tagged resources despite the fact // it no longer exists. - if strings.Contains(HandleErrorCode(err), "NoSuchHostedZone") { + if strings.Contains(handleErrorCode(err), "NoSuchHostedZone") { return nil } return err @@ -850,7 +894,7 @@ func deleteFileSystem(ctx context.Context, client *efs.Client, fsid string, logg _, err = client.DeleteFileSystem(ctx, &efs.DeleteFileSystemInput{FileSystemId: aws.String(fsid)}) if err != nil { - if strings.Contains(HandleErrorCode(err), "FileSystemNotFound") { + if strings.Contains(handleErrorCode(err), "FileSystemNotFound") { return nil } return err @@ -909,7 +953,7 @@ func deleteAccessPoint(ctx context.Context, client *efs.Client, id string, logge logger = logger.WithField("AccessPoint ID", id) _, err := client.DeleteAccessPoint(ctx, &efs.DeleteAccessPointInput{AccessPointId: aws.String(id)}) if err != nil { - if strings.Contains(HandleErrorCode(err), "AccessPointNotFound") { + if strings.Contains(handleErrorCode(err), "AccessPointNotFound") { return nil } return err @@ -923,7 +967,7 @@ func deleteMountTarget(ctx context.Context, client *efs.Client, id string, logge logger = logger.WithField("Mount Target ID", id) _, err := client.DeleteMountTarget(ctx, &efs.DeleteMountTargetInput{MountTargetId: aws.String(id)}) if err != nil { - if strings.Contains(HandleErrorCode(err), "MountTargetNotFound") { + if strings.Contains(handleErrorCode(err), "MountTargetNotFound") { return nil } return err diff --git a/pkg/destroy/aws/ec2helpers.go b/pkg/destroy/aws/ec2helpers.go index 594fccda01..525ce46a2d 100644 --- a/pkg/destroy/aws/ec2helpers.go +++ b/pkg/destroy/aws/ec2helpers.go @@ -12,7 +12,6 @@ import ( elb "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" iamv2 "github.com/aws/aws-sdk-go-v2/service/iam" - "github.com/aws/aws-sdk-go/aws/session" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/sets" @@ -83,7 +82,7 @@ func findEC2Instances(ctx context.Context, ec2Client *ec2v2.Client, deleted sets } // DeleteEC2Instances terminates all EC2 instances found. -func (o *ClusterUninstaller) DeleteEC2Instances(ctx context.Context, awsSession *session.Session, toDelete sets.Set[string], deleted sets.Set[string], tracker *ErrorTracker) error { +func (o *ClusterUninstaller) DeleteEC2Instances(ctx context.Context, toDelete sets.Set[string], deleted sets.Set[string], tracker *ErrorTracker) error { lastTerminateTime := time.Now() err := wait.PollUntilContextCancel( ctx, @@ -103,7 +102,7 @@ func (o *ClusterUninstaller) DeleteEC2Instances(ctx context.Context, awsSession instancesToDelete = instancesNotTerminated lastTerminateTime = time.Now() } - newlyDeleted, err := o.DeleteResources(ctx, awsSession, instancesToDelete, tracker) + newlyDeleted, err := o.DeleteResources(ctx, instancesToDelete, tracker) // Delete from the resources-to-delete set so that the current state of the resources to delete can be // returned if the context is completed. toDelete = toDelete.Difference(newlyDeleted) @@ -173,7 +172,7 @@ func deleteEC2DHCPOptions(ctx context.Context, client *ec2v2.Client, id string, DhcpOptionsId: &id, }) if err != nil { - if HandleErrorCode(err) == "InvalidDhcpOptions.NotFound" { + if handleErrorCode(err) == "InvalidDhcpOptions.NotFound" { return nil } return err @@ -190,7 +189,7 @@ func deleteEC2Image(ctx context.Context, client *ec2v2.Client, id string, logger ImageIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidAMI.NotFound" { + if handleErrorCode(err) == "InvalidAMI.NotFound" { return nil } return err @@ -217,7 +216,7 @@ func deleteEC2Image(ctx context.Context, client *ec2v2.Client, id string, logger ImageId: &id, }) if err != nil { - if HandleErrorCode(err) == "InvalidAMI.NotFound" { + if handleErrorCode(err) == "InvalidAMI.NotFound" { return nil } return err @@ -232,7 +231,7 @@ func deleteEC2ElasticIP(ctx context.Context, client *ec2v2.Client, id string, lo AllocationId: aws.String(id), }) if err != nil { - if HandleErrorCode(err) == "InvalidAllocation.NotFound" { + if handleErrorCode(err) == "InvalidAllocation.NotFound" { return nil } return err @@ -247,7 +246,7 @@ func terminateEC2Instance(ctx context.Context, ec2Client *ec2v2.Client, iamClien InstanceIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidInstance.NotFound" { + if handleErrorCode(err) == "InvalidInstance.NotFound" { return nil } return err @@ -301,7 +300,7 @@ func deleteEC2InternetGateway(ctx context.Context, client *ec2v2.Client, id stri }) if err == nil { logger.WithField("vpc", *vpc.VpcId).Debug("Detached") - } else if HandleErrorCode(err) == "Gateway.NotAttached" { + } else if handleErrorCode(err) == "Gateway.NotAttached" { return nil } } @@ -323,7 +322,7 @@ func deleteEC2CarrierGateway(ctx context.Context, client *ec2v2.Client, id strin CarrierGatewayId: &id, }) if err != nil { - if HandleErrorCode(err) == "InvalidCarrierGateway.NotFound" { + if handleErrorCode(err) == "InvalidCarrierGateway.NotFound" { return nil } return err @@ -338,7 +337,7 @@ func deleteEC2NATGateway(ctx context.Context, client *ec2v2.Client, id string, l NatGatewayId: aws.String(id), }) if err != nil { - if HandleErrorCode(err) == "NatGateway.NotFound" { + if handleErrorCode(err) == "NatGateway.NotFound" { return nil } return err @@ -391,7 +390,7 @@ func deleteEC2PlacementGroup(ctx context.Context, client *ec2v2.Client, id strin GroupIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidPlacementGroup.NotFound" { + if handleErrorCode(err) == "InvalidPlacementGroup.NotFound" { return nil } return err @@ -414,7 +413,7 @@ func deleteEC2RouteTable(ctx context.Context, client *ec2v2.Client, id string, l RouteTableIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidRouteTableID.NotFound" { + if handleErrorCode(err) == "InvalidRouteTableID.NotFound" { return nil } return err @@ -544,7 +543,7 @@ func deleteEC2SecurityGroup(ctx context.Context, client *ec2v2.Client, id string GroupIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidGroup.NotFound" { + if handleErrorCode(err) == "InvalidGroup.NotFound" { return nil } return err @@ -594,7 +593,7 @@ func deleteEC2SecurityGroupObject(ctx context.Context, client *ec2v2.Client, gro GroupId: group.GroupId, }) if err != nil { - if HandleErrorCode(err) == "InvalidGroup.NotFound" { + if handleErrorCode(err) == "InvalidGroup.NotFound" { return nil } return err @@ -648,7 +647,7 @@ func deleteEC2Snapshot(ctx context.Context, client *ec2v2.Client, id string, log SnapshotId: &id, }) if err != nil { - if HandleErrorCode(err) == "InvalidSnapshot.NotFound" { + if handleErrorCode(err) == "InvalidSnapshot.NotFound" { return nil } return err @@ -663,7 +662,7 @@ func deleteEC2NetworkInterface(ctx context.Context, client *ec2v2.Client, id str NetworkInterfaceId: aws.String(id), }) if err != nil { - if HandleErrorCode(err) == "InvalidNetworkInterfaceID.NotFound" { + if handleErrorCode(err) == "InvalidNetworkInterfaceID.NotFound" { return nil } return err @@ -714,7 +713,7 @@ func deleteEC2Subnet(ctx context.Context, client *ec2v2.Client, id string, logge SubnetId: aws.String(id), }) if err != nil { - if HandleErrorCode(err) == "InvalidSubnetID.NotFound" { + if handleErrorCode(err) == "InvalidSubnetID.NotFound" { return nil } return err @@ -766,7 +765,7 @@ func deleteEC2Volume(ctx context.Context, client *ec2v2.Client, id string, logge VolumeId: aws.String(id), }) if err != nil { - if HandleErrorCode(err) == "InvalidVolume.NotFound" { + if handleErrorCode(err) == "InvalidVolume.NotFound" { return nil } return err @@ -845,7 +844,7 @@ func deleteEC2VPCEndpointsByVPC(ctx context.Context, client *ec2v2.Client, vpc s for _, endpoint := range response.VpcEndpoints { err := deleteEC2VPCEndpoint(ctx, client, *endpoint.VpcEndpointId, logger.WithField("VPC endpoint", *endpoint.VpcEndpointId)) if err != nil { - if HandleErrorCode(err) == "InvalidVpcID.NotFound" { + if handleErrorCode(err) == "InvalidVpcID.NotFound" { return nil } return err @@ -860,7 +859,7 @@ func deleteEC2VPCPeeringConnection(ctx context.Context, client *ec2v2.Client, id VpcPeeringConnectionId: &id, }) if err != nil { - if HandleErrorCode(err) == "InvalidVpcPeeringConnection.NotFound" { + if handleErrorCode(err) == "InvalidVpcPeeringConnection.NotFound" { return nil } return errors.Wrapf(err, "cannot delete VPC Peering Connection %s", id) @@ -907,7 +906,7 @@ func deleteEC2VPCEndpointService(ctx context.Context, client *ec2v2.Client, id s ServiceIds: []string{id}, }) if err != nil { - if HandleErrorCode(err) == "InvalidVpcEndpointService.NotFound" { + if handleErrorCode(err) == "InvalidVpcEndpointService.NotFound" { return nil } return errors.Wrapf(err, "cannot delete VPC Endpoint Service %s", id) diff --git a/pkg/destroy/aws/elbhelpers.go b/pkg/destroy/aws/elbhelpers.go index 95d9601ab2..7cdab3b084 100644 --- a/pkg/destroy/aws/elbhelpers.go +++ b/pkg/destroy/aws/elbhelpers.go @@ -111,7 +111,7 @@ func deleteElasticLoadBalancerListener(ctx context.Context, client *elbapiv2.Cli ListenerArn: aws.String(arn.String()), }) if err != nil { - if strings.Contains(HandleErrorCode(err), "ListenerNotFound") { + if strings.Contains(handleErrorCode(err), "ListenerNotFound") { logger.Info("Not found or already deleted") return nil } diff --git a/pkg/destroy/aws/errors.go b/pkg/destroy/aws/errors.go index a2e161a2fa..b70b3e4636 100644 --- a/pkg/destroy/aws/errors.go +++ b/pkg/destroy/aws/errors.go @@ -5,8 +5,8 @@ import ( "github.com/pkg/errors" ) -// HandleErrorCode takes the error and extracts the error code if it was successfully cast as an API Error. -func HandleErrorCode(err error) string { +// handleErrorCode takes the error and extracts the error code if it was successfully cast as an API Error. +func handleErrorCode(err error) string { var apiErr smithy.APIError if errors.As(err, &apiErr) { return apiErr.ErrorCode() diff --git a/pkg/destroy/aws/iamhelpers.go b/pkg/destroy/aws/iamhelpers.go index b7215c48d7..6ae543c8b7 100644 --- a/pkg/destroy/aws/iamhelpers.go +++ b/pkg/destroy/aws/iamhelpers.go @@ -46,7 +46,7 @@ func (search *IamRoleSearch) find(ctx context.Context) (arns []string, names []s response, err := search.Client.ListRoleTags(ctx, &iamv2.ListRoleTagsInput{RoleName: role.RoleName}) if err != nil { switch { - case strings.Contains(HandleErrorCode(err), "NoSuchEntity"): + case strings.Contains(handleErrorCode(err), "NoSuchEntity"): // The role does not exist. // Ignore this IAM Role and donot report this error via // lastError @@ -195,7 +195,7 @@ func deleteIAMInstanceProfileByName(ctx context.Context, client *iamv2.Client, n InstanceProfileName: name, }) if err != nil { - if strings.Contains(HandleErrorCode(err), "NoSuchEntity") { + if strings.Contains(handleErrorCode(err), "NoSuchEntity") { return nil } return err @@ -218,7 +218,7 @@ func deleteIAMInstanceProfile(ctx context.Context, client *iamv2.Client, profile InstanceProfileName: &name, }) if err != nil { - if strings.Contains(HandleErrorCode(err), "NoSuchEntity") { + if strings.Contains(handleErrorCode(err), "NoSuchEntity") { return nil } return err diff --git a/pkg/destroy/aws/shared.go b/pkg/destroy/aws/shared.go index f40a47a1ea..6d152a6429 100644 --- a/pkg/destroy/aws/shared.go +++ b/pkg/destroy/aws/shared.go @@ -6,15 +6,13 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" + tagtypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" "github.com/aws/aws-sdk-go-v2/service/route53" route53types "github.com/aws/aws-sdk-go-v2/service/route53/types" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - //"github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" @@ -22,12 +20,11 @@ import ( func (o *ClusterUninstaller) removeSharedTags( ctx context.Context, - session *session.Session, - tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, + tagClients []*resourcegroupstaggingapi.Client, tracker *ErrorTracker, ) error { for _, key := range o.clusterOwnedKeys() { - if err := o.removeSharedTag(ctx, session, tagClients, key, tracker); err != nil { + if err := o.removeSharedTag(ctx, tagClients, key, tracker); err != nil { return err } } @@ -50,64 +47,67 @@ func (o *ClusterUninstaller) clusterOwnedKeys() []string { return keys } -func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, session *session.Session, tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, key string, tracker *ErrorTracker) error { +func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, tagClients []*resourcegroupstaggingapi.Client, key string, tracker *ErrorTracker) error { const sharedValue = "shared" request := &resourcegroupstaggingapi.UntagResourcesInput{ - TagKeys: []*string{aws.String(key)}, + TagKeys: []string{key}, } removed := map[string]struct{}{} - tagClients = append([]*resourcegroupstaggingapi.ResourceGroupsTaggingAPI(nil), tagClients...) + tagClients = append([]*resourcegroupstaggingapi.Client(nil), tagClients...) for len(tagClients) > 0 { nextTagClients := tagClients[:0] for _, tagClient := range tagClients { - o.Logger.Debugf("Search for and remove tags in %s matching %s: shared", *tagClient.Config.Region, key) + var lastErr error + o.Logger.Debugf("Search for and remove tags in %s matching %s: shared", tagClient.Options().Region, key) var arns []string - err := tagClient.GetResourcesPagesWithContext( - ctx, - &resourcegroupstaggingapi.GetResourcesInput{TagFilters: []*resourcegroupstaggingapi.TagFilter{{ + paginator := resourcegroupstaggingapi.NewGetResourcesPaginator(tagClient, &resourcegroupstaggingapi.GetResourcesInput{ + TagFilters: []tagtypes.TagFilter{{ Key: aws.String(key), - Values: []*string{aws.String(sharedValue)}, - }}}, - func(results *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { - for _, resource := range results.ResourceTagMappingList { - arnString := aws.StringValue(resource.ResourceARN) - logger := o.Logger.WithField("arn", arnString) - parsedARN, err := arn.Parse(arnString) - if err != nil { - logger.WithError(err).Debug("could not parse ARN") - continue - } - if _, ok := removed[arnString]; !ok { - if err := o.cleanSharedARN(ctx, parsedARN, logger); err != nil { - tracker.suppressWarning(arnString, err, logger) - if err := ctx.Err(); err != nil { - return false - } - continue - } - arns = append(arns, arnString) - } - } + Values: []string{sharedValue}, + }}}) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + o.Logger.Debugf("failed to get resources: %v", err) + lastErr = err + break + } - return !lastPage - }, - ) - if err != nil { - err2 := errors.Wrap(err, "get tagged resources") - o.Logger.Info(err2) - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case resourcegroupstaggingapi.ErrorCodeInvalidParameterException: + for _, resource := range page.ResourceTagMappingList { + arnString := *resource.ResourceARN + logger := o.Logger.WithField("arn", arnString) + parsedARN, err := arn.Parse(arnString) + if err != nil { + logger.WithError(err).Debug("could not parse ARN") continue } + if _, ok := removed[arnString]; !ok { + if err := o.cleanSharedARN(ctx, parsedARN, logger); err != nil { + tracker.suppressWarning(arnString, err, logger) + if err := ctx.Err(); err != nil { + lastErr = fmt.Errorf("failed to remove tag %q: %w", key, err) + } + continue + } + arns = append(arns, arnString) + } + } + } + + if lastErr != nil { + o.Logger.Infof("failed to get tagged resources: %v", lastErr) + var invalidParameter tagtypes.InvalidParameterException + if errors.As(lastErr, &invalidParameter) { + continue } nextTagClients = append(nextTagClients, tagClient) continue } + if len(arns) == 0 { - o.Logger.Debugf("No matches in %s for %s: shared, removing client", *tagClient.Config.Region, key) + o.Logger.Debugf("No matches in %s for %s: shared, removing client", o.Region, key) continue } // appending the tag client here but it needs to be removed if there is a InvalidParameterException when trying to @@ -115,15 +115,13 @@ func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, session *sessi nextTagClients = append(nextTagClients, tagClient) for i := 0; i < len(arns); i += 20 { - request.ResourceARNList = make([]*string, 0, 20) + request.ResourceARNList = make([]string, 0, 20) for j := 0; i+j < len(arns) && j < 20; j++ { - request.ResourceARNList = append(request.ResourceARNList, aws.String(arns[i+j])) + request.ResourceARNList = append(request.ResourceARNList, arns[i+j]) } - result, err := tagClient.UntagResourcesWithContext(ctx, request) + result, err := tagClient.UntagResources(ctx, request) if err != nil { - var awsErr awserr.Error - ok := errors.As(err, &awsErr) - if ok && awsErr.Code() == resourcegroupstaggingapi.ErrorCodeInvalidParameterException { + if strings.Contains(handleErrorCode(err), "InvalidParameter") { nextTagClients = nextTagClients[:len(nextTagClients)-1] } err = errors.Wrap(err, "untag shared resources") @@ -131,19 +129,18 @@ func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, session *sessi continue } for _, arn := range request.ResourceARNList { - if info, failed := result.FailedResourcesMap[*arn]; failed { - o.Logger.WithField("arn", *arn).Infof("Failed to remove tag %s: shared; error=%s", key, *info.ErrorMessage) + if info, failed := result.FailedResourcesMap[arn]; failed { + o.Logger.WithField("arn", arn).Infof("Failed to remove tag %s: shared; error=%s", key, *info.ErrorMessage) continue } - o.Logger.WithField("arn", *arn).Infof("Removed tag %s: shared", key) - removed[*arn] = exists + o.Logger.WithField("arn", arn).Infof("Removed tag %s: shared", key) + removed[arn] = exists } } } tagClients = nextTagClients } - iamClient := iam.New(session) iamRoleSearch := &IamRoleSearch{ Client: o.IAMClient, Filters: []Filter{{key: sharedValue}}, @@ -163,9 +160,9 @@ func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, session *sessi o.Logger.Debugf("Removing the shared tag from the %q IAM role", role) input := &iam.UntagRoleInput{ RoleName: &role, - TagKeys: []*string{&key}, + TagKeys: []string{key}, } - if _, err := iamClient.UntagRoleWithContext(ctx, input); err != nil { + if _, err := o.IAMClient.UntagRole(ctx, input); err != nil { done = false o.Logger.Infof("Could not remove the shared tag from the %q IAM role: %v", role, err) } @@ -228,14 +225,14 @@ func (o *ClusterUninstaller) cleanSharedHostedZone(ctx context.Context, id strin for _, recordSet := range page.ResourceRecordSets { // skip record sets that are not part of the cluster - name := aws.StringValue(recordSet.Name) + name := *recordSet.Name if !strings.HasSuffix(name, dottedClusterDomain) { continue } if len(name) == len(dottedClusterDomain) { continue } - recordsetFields := logrus.Fields{"recordset": fmt.Sprintf("%s (%s)", aws.StringValue(recordSet.Name), recordSet.Type)} + recordsetFields := logrus.Fields{"recordset": fmt.Sprintf("%s (%s)", *recordSet.Name, recordSet.Type)} // delete any matching record sets in the public hosted zone if publicZoneID != "" { publicZoneLogger := logger.WithField("id", publicZoneID) @@ -283,8 +280,7 @@ func deleteMatchingRecordSetInPublicZone(ctx context.Context, client *route53.Cl return nil } matchingRecordSet := out.ResourceRecordSets[0] - if aws.StringValue(matchingRecordSet.Name) != aws.StringValue(recordSet.Name) || - matchingRecordSet.Type != recordSet.Type { + if *matchingRecordSet.Name != *recordSet.Name || matchingRecordSet.Type != recordSet.Type { return nil } return deleteRoute53RecordSet(ctx, client, zoneID, &matchingRecordSet, logger)