From 32356ddc99c5bdd3a45644542ce6b0a3f2812804 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Mon, 7 Oct 2019 16:59:40 -0700 Subject: [PATCH] pkg/types/aws/platform: Add Subnets property This allows users to feed in prexisting subnets. I've also added classification logic copied from the upstream Kubernetes AWS cloud provider for categorizing private vs. public subnets. There's no explicit install-config field for the VPC; we'll extract that from the given subnets. populateSubnets is a bit heavy if all you need is the VPC, but Abhinav prefers it [1], it would save future public/private subnet lookup by warming those caches, and we'll only call Metadata.AWS late in pkg/asset/cluster/tfvars.go (via a future commit), so the VPC cache-warming logic doesn't get called at the moment anyway. There's no verification yet; we'll get to that in follow-up work. More on the DescribeSubnetsPagesWithContext FIXME in 37a7f49c77 (pkg/destroy/aws: Delete subnets by VPC, 2019-08-13, #2214). [1]: https://github.com/openshift/installer/pull/2477#discussion_r334701272 --- pkg/asset/installconfig/aws/metadata.go | 75 ++++++++++- pkg/asset/installconfig/aws/subnet.go | 151 +++++++++++++++++++++++ pkg/asset/installconfig/installconfig.go | 2 +- pkg/types/aws/platform.go | 5 + 4 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 pkg/asset/installconfig/aws/subnet.go diff --git a/pkg/asset/installconfig/aws/metadata.go b/pkg/asset/installconfig/aws/metadata.go index 1e273e144e..ca41650e83 100644 --- a/pkg/asset/installconfig/aws/metadata.go +++ b/pkg/asset/installconfig/aws/metadata.go @@ -14,13 +14,17 @@ import ( type Metadata struct { session *session.Session availabilityZones []string + privateSubnets map[string]Subnet + publicSubnets map[string]Subnet region string + subnets []string + vpc string mutex sync.Mutex } // NewMetadata initializes a new Metadata object. -func NewMetadata(region string) *Metadata { - return &Metadata{region: region} +func NewMetadata(region string, subnets []string) *Metadata { + return &Metadata{region: region, subnets: subnets} } // Session holds an AWS session which can be used for AWS API calls @@ -63,3 +67,70 @@ func (m *Metadata) AvailabilityZones(ctx context.Context) ([]string, error) { return m.availabilityZones, nil } + +// PrivateSubnets retrieves subnet metadata indexed by subnet ID, for +// subnets that the cloud-provider logic considers to be private +// (i.e. not public). +func (m *Metadata) PrivateSubnets(ctx context.Context) (map[string]Subnet, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + err := m.populateSubnets(ctx) + if err != nil { + return nil, err + } + + return m.privateSubnets, nil +} + +// PublicSubnets retrieves subnet metadata indexed by subnet ID, for +// subnets that the cloud-provider logic considers to be public +// (e.g. with suitable routing for hosting public load balancers). +func (m *Metadata) PublicSubnets(ctx context.Context) (map[string]Subnet, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + err := m.populateSubnets(ctx) + if err != nil { + return nil, err + } + + return m.publicSubnets, nil +} + +func (m *Metadata) populateSubnets(ctx context.Context) error { + if len(m.publicSubnets) > 0 || len(m.privateSubnets) > 0 { + return nil + } + + if len(m.subnets) == 0 { + return errors.New("no subnets configured") + } + + session, err := m.unlockedSession(ctx) + if err != nil { + return err + } + + m.vpc, m.privateSubnets, m.publicSubnets, err = subnets(ctx, session, m.subnets) + return err +} + +// VPC retrieves the VPC ID containing PublicSubnets and PrivateSubnets. +func (m *Metadata) VPC(ctx context.Context) (string, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if m.vpc == "" { + if len(m.subnets) == 0 { + return "", errors.New("cannot calculate VPC without configured subnets") + } + + err := m.populateSubnets(ctx) + if err != nil { + return "", err + } + } + + return m.vpc, nil +} diff --git a/pkg/asset/installconfig/aws/subnet.go b/pkg/asset/installconfig/aws/subnet.go new file mode 100644 index 0000000000..a9356e25bc --- /dev/null +++ b/pkg/asset/installconfig/aws/subnet.go @@ -0,0 +1,151 @@ +package aws + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// Subnet holds metadata for a subnet. +type Subnet struct { + // ARN is the subnet's Amazon Resource Name. + ARN string + + // Zone is the subnet's availability zone. + Zone string +} + +// subnets retrieves metadata for the given subnet(s). +func subnets(ctx context.Context, session *session.Session, ids []string) (vpc string, private map[string]Subnet, public map[string]Subnet, err error) { + metas := make(map[string]Subnet, len(ids)) + private = map[string]Subnet{} + public = map[string]Subnet{} + var vpcFromSubnet string + client := ec2.New(session) + + idPointers := make([]*string, len(ids)) + for _, id := range ids { + idPointers = append(idPointers, aws.String(id)) + } + results, err := client.DescribeSubnetsWithContext( // FIXME: port to DescribeSubnetsPagesWithContext once we bump our vendored AWS package past v1.19.30 + ctx, + &ec2.DescribeSubnetsInput{SubnetIds: idPointers}, + ) + if err != nil { + return vpc, nil, nil, errors.Wrap(err, "describing subnets") + } + for _, subnet := range results.Subnets { + if subnet.SubnetId == nil { + continue + } + if subnet.SubnetArn == nil { + return vpc, nil, nil, errors.Errorf("%s has no ARN", *subnet.SubnetId) + } + if subnet.VpcId == nil { + return vpc, nil, nil, errors.Errorf("%s has no VPC", *subnet.SubnetId) + } + if subnet.AvailabilityZone == nil { + return vpc, nil, nil, errors.Errorf("%s has no availability zone", *subnet.SubnetId) + } + + if vpc == "" { + vpc = *subnet.VpcId + vpcFromSubnet = *subnet.SubnetId + } else if *subnet.VpcId != vpc { + return vpc, nil, nil, errors.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, vpc) + } + + metas[*subnet.SubnetId] = Subnet{ + ARN: *subnet.SubnetArn, + Zone: *subnet.AvailabilityZone, + } + } + + var routeTables []*ec2.RouteTable + err = client.DescribeRouteTablesPagesWithContext( + ctx, + &ec2.DescribeRouteTablesInput{ + Filters: []*ec2.Filter{{ + Name: aws.String("vpc-id"), + Values: []*string{aws.String(vpc)}, + }}, + }, + func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool { + routeTables = append(routeTables, results.RouteTables...) + return !lastPage + }, + ) + if err != nil { + return vpc, nil, nil, errors.Wrap(err, "describing route tables") + } + + for _, id := range ids { + meta, ok := metas[id] + if !ok { + return vpc, nil, nil, errors.Errorf("failed to find %s", id) + } + isPublic, err := isSubnetPublic(routeTables, id) + if err != nil { + return vpc, nil, nil, err + } + if isPublic { + public[id] = meta + } else { + private[id] = meta + } + } + + return vpc, private, public, nil +} + +// https://github.com/kubernetes/kubernetes/blob/9f036cd43d35a9c41d7ac4ca82398a6d0bef957b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L3376-L3419 +func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { + var subnetTable *ec2.RouteTable + for _, table := range rt { + for _, assoc := range table.Associations { + if aws.StringValue(assoc.SubnetId) == subnetID { + subnetTable = table + break + } + } + } + + if subnetTable == nil { + // If there is no explicit association, the subnet will be implicitly + // associated with the VPC's main routing table. + for _, table := range rt { + for _, assoc := range table.Associations { + if aws.BoolValue(assoc.Main) == true { + logrus.Debugf("Assuming implicit use of main routing table %s for %s", + aws.StringValue(table.RouteTableId), subnetID) + subnetTable = table + break + } + } + } + } + + if subnetTable == nil { + return false, fmt.Errorf("could not locate routing table for %s", subnetID) + } + + for _, route := range subnetTable.Routes { + // There is no direct way in the AWS API to determine if a subnet is public or private. + // A public subnet is one which has an internet gateway route + // we look for the gatewayId and make sure it has the prefix of igw to differentiate + // from the default in-subnet route which is called "local" + // or other virtual gateway (starting with vgv) + // or vpc peering connections (starting with pcx). + if strings.HasPrefix(aws.StringValue(route.GatewayId), "igw") { + return true, nil + } + } + + return false, nil +} diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index 997df70fca..62dda256cc 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -128,7 +128,7 @@ func (a *InstallConfig) finish(filename string) error { defaults.SetInstallConfigDefaults(a.Config) if a.Config.AWS != nil { - a.AWS = aws.NewMetadata(a.Config.Platform.AWS.Region) + a.AWS = aws.NewMetadata(a.Config.Platform.AWS.Region, a.Config.Platform.AWS.Subnets) } if err := validation.ValidateInstallConfig(a.Config, openstackvalidation.NewValidValuesFetcher()).ToAggregate(); err != nil { diff --git a/pkg/types/aws/platform.go b/pkg/types/aws/platform.go index 93153f4b8c..45d0911ec5 100644 --- a/pkg/types/aws/platform.go +++ b/pkg/types/aws/platform.go @@ -10,6 +10,11 @@ type Platform struct { // Region specifies the AWS region where the cluster will be created. Region string `json:"region"` + // Subnets specifies existing subnets (by ID) where cluster + // resources will be created. Leave unset to have the installer + // create subnets in a new VPC on your behalf. + Subnets []string `json:"subnets,omitempty"` + // UserTags additional keys and values that the installer will add // as tags to all resources that it creates. Resources created by the // cluster itself may not include these tags.