From 878bd5686b8a679d8cde43774e212b34fea3ae51 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 18 Mar 2025 18:25:52 -0700 Subject: [PATCH 1/2] validations: add validations for subnets field with AWS API This include validations for the vpc.subnets field with AWS API to conform to the specifications for subnet role assignment in the case of BYO subnets. The validation criteria can be found at jira ticket [0] and enhancement proposal [1]. For automatic role selection (i.e. no subnets have roles assigned), the installer rejects BYO VPC with untagged subnets (i.e. those without tag kubernetes.io/cluster/) and suggests users to add a tag kubernetes.io/cluster/unmanaged to those subnets. References [0] https://issues.redhat.com/browse/CORS-3870 [1] https://github.com/openshift/enhancements/blob/master/enhancements/installer/aws-lb-subnet-selection.md#installer-validation-rules --- pkg/asset/installconfig/aws/metadata.go | 91 +++-- pkg/asset/installconfig/aws/subnet.go | 234 ++++++----- pkg/asset/installconfig/aws/tags.go | 44 +++ pkg/asset/installconfig/aws/validation.go | 371 +++++++++++++++--- .../installconfig/aws/validation_test.go | 44 ++- 5 files changed, 604 insertions(+), 180 deletions(-) create mode 100644 pkg/asset/installconfig/aws/tags.go diff --git a/pkg/asset/installconfig/aws/metadata.go b/pkg/asset/installconfig/aws/metadata.go index 5e874de26d..ad2ed1aaea 100644 --- a/pkg/asset/installconfig/aws/metadata.go +++ b/pkg/asset/installconfig/aws/metadata.go @@ -26,15 +26,14 @@ type Metadata struct { availabilityZones []string availableRegions []string edgeZones []string - privateSubnets Subnets - publicSubnets Subnets - edgeSubnets Subnets + subnets SubnetGroups + vpcSubnets SubnetGroups vpc string instanceTypes map[string]InstanceType - Region string `json:"region,omitempty"` - Subnets []typesaws.Subnet `json:"subnets,omitempty"` - Services []typesaws.ServiceEndpoint `json:"services,omitempty"` + Region string `json:"region,omitempty"` + ProvidedSubnets []typesaws.Subnet `json:"subnets,omitempty"` + Services []typesaws.ServiceEndpoint `json:"services,omitempty"` ec2Client *ec2.Client @@ -43,7 +42,7 @@ type Metadata struct { // NewMetadata initializes a new Metadata object. func NewMetadata(region string, subnets []typesaws.Subnet, services []typesaws.ServiceEndpoint) *Metadata { - return &Metadata{Region: region, Subnets: subnets, Services: services} + return &Metadata{Region: region, ProvidedSubnets: subnets, Services: services} } // Session holds an AWS session which can be used for AWS API calls @@ -174,7 +173,7 @@ func (m *Metadata) EdgeSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Edge Subnets: %w", err) } - return m.edgeSubnets, nil + return m.subnets.Edge, nil } // SetZoneAttributes retrieves AWS Zone attributes and update required fields in zones. @@ -239,7 +238,7 @@ func (m *Metadata) PrivateSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Private Subnets: %w", err) } - return m.privateSubnets, nil + return m.subnets.Private, nil } // PublicSubnets retrieves subnet metadata indexed by subnet ID, for @@ -250,7 +249,29 @@ func (m *Metadata) PublicSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Public Subnets: %w", err) } - return m.publicSubnets, nil + return m.subnets.Public, nil +} + +// Subnets retrieves a group of subnet metadata that is indexed by subnet ID for all provided subnets. +// This includes private, public and edge subnets. +func (m *Metadata) Subnets(ctx context.Context) (SubnetGroups, error) { + err := m.populateSubnets(ctx) + if err != nil { + return m.subnets, fmt.Errorf("error retrieving all Subnets: %w", err) + } + return m.subnets, nil +} + +// VPCSubnets retrieves a group of all subnet metadata that is indexed by subnet ID in the VPC of the provided subnets. +// These include cluster subnets (i.e. provided in the installconfig) and potentially other non-cluster subnets in the VPC. +// +// This func is only used for validations. Use func Subnets to select only cluster subnets. +func (m *Metadata) VPCSubnets(ctx context.Context) (SubnetGroups, error) { + err := m.populateVPCSubnets(ctx) + if err != nil { + return m.vpcSubnets, fmt.Errorf("error retrieving Subnets in VPC: %w", err) + } + return m.vpcSubnets, nil } // VPC retrieves the VPC ID containing PublicSubnets and PrivateSubnets. @@ -269,49 +290,75 @@ func (m *Metadata) SubnetByID(ctx context.Context, subnetID string) (subnet Subn return subnet, fmt.Errorf("error retrieving subnet for ID %s: %w", subnetID, err) } - if subnet, ok := m.privateSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Private[subnetID]; ok { return subnet, nil } - if subnet, ok := m.publicSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Public[subnetID]; ok { return subnet, nil } - if subnet, ok := m.edgeSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Edge[subnetID]; ok { return subnet, nil } return subnet, fmt.Errorf("no subnet found for ID %s", subnetID) } +// populateSubnets retrieves metadata for provided subnets. func (m *Metadata) populateSubnets(ctx context.Context) error { m.mutex.Lock() defer m.mutex.Unlock() - if len(m.Subnets) == 0 { + if len(m.ProvidedSubnets) == 0 { return errors.New("no subnets configured") } - if m.vpc != "" || len(m.privateSubnets) > 0 || len(m.publicSubnets) > 0 || len(m.edgeSubnets) > 0 { + subnetGroups := m.subnets + if m.vpc != "" || len(subnetGroups.Private) > 0 || len(subnetGroups.Public) > 0 || len(subnetGroups.Edge) > 0 { // Call to populate subnets has already happened return nil } - session, err := m.unlockedSession(ctx) + client, err := m.EC2Client(ctx) if err != nil { return err } - subnetIDs := make([]string, len(m.Subnets)) - for i, subnet := range m.Subnets { + subnetIDs := make([]string, len(m.ProvidedSubnets)) + for i, subnet := range m.ProvidedSubnets { subnetIDs[i] = string(subnet.ID) } - sb, err := subnets(ctx, session, m.Region, subnetIDs) + sb, err := subnets(ctx, client, subnetIDs, "") m.vpc = sb.VPC - m.privateSubnets = sb.Private - m.publicSubnets = sb.Public - m.edgeSubnets = sb.Edge + m.subnets = sb + return err +} + +// populateVPCSubnets retrieves metadata for all subnets in the VPC of provided subnets. +func (m *Metadata) populateVPCSubnets(ctx context.Context) error { + // we need to populate provided subnets to get the VPC ID. + if err := m.populateSubnets(ctx); err != nil { + return err + } + + m.mutex.Lock() + defer m.mutex.Unlock() + + vpcSubnetGroups := m.vpcSubnets + if len(vpcSubnetGroups.Private) > 0 || len(vpcSubnetGroups.Public) > 0 || len(vpcSubnetGroups.Edge) > 0 { + // Call to populate subnets has already happened + return nil + } + + client, err := m.EC2Client(ctx) + if err != nil { + return err + } + + sb, err := subnets(ctx, client, nil, m.vpc) + m.vpcSubnets = sb return err } diff --git a/pkg/asset/installconfig/aws/subnet.go b/pkg/asset/installconfig/aws/subnet.go index a1fd38095d..601b6c4c18 100644 --- a/pkg/asset/installconfig/aws/subnet.go +++ b/pkg/asset/installconfig/aws/subnet.go @@ -3,12 +3,13 @@ package aws import ( "context" "fmt" + "maps" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/sirupsen/logrus" + "k8s.io/utils/ptr" typesaws "github.com/openshift/installer/pkg/types/aws" ) @@ -29,11 +30,23 @@ type Subnet struct { // Public is the flag to define the subnet public. Public bool + + // Tags is the map of the subnet's tags. + Tags Tags } // Subnets is the map for the Subnet metadata indexed by subnetID. type Subnets map[string]Subnet +// IDs returns the subnet IDs (i.e. map keys) in the Subnets. +func (sns Subnets) IDs() []string { + subnetIDs := make([]string, 0) + for id := range sns { + subnetIDs = append(subnetIDs, id) + } + return subnetIDs +} + // SubnetsByZone is the map for the Subnet metadata indexed by zone. type SubnetsByZone map[string]Subnet @@ -45,92 +58,82 @@ type SubnetGroups struct { VPC string } -// subnets retrieves metadata for the given subnet(s). -func subnets(ctx context.Context, session *session.Session, region string, ids []string) (subnetGroups SubnetGroups, err error) { - metas := make(Subnets, len(ids)) - zoneNames := make([]*string, len(ids)) - availabilityZones := make(map[string]*ec2.AvailabilityZone, len(ids)) - subnetGroups = SubnetGroups{ - Public: make(Subnets, len(ids)), - Private: make(Subnets, len(ids)), - Edge: make(Subnets, len(ids)), +// subnets retrieves metadata for the given subnet(s) or VPC. +func subnets(ctx context.Context, client *ec2.Client, subnetIDs []string, vpcID string) (SubnetGroups, error) { + metas := make(Subnets, len(subnetIDs)) + zoneNames := make([]string, 0) + availabilityZones := make(map[string]ec2types.AvailabilityZone, len(subnetIDs)) + subnetGroups := SubnetGroups{ + Public: make(Subnets, len(subnetIDs)), + Private: make(Subnets, len(subnetIDs)), + Edge: make(Subnets, len(subnetIDs)), } - var vpcFromSubnet string - client := ec2.New(session, aws.NewConfig().WithRegion(region)) - - idPointers := make([]*string, 0, len(ids)) - for _, id := range ids { - idPointers = append(idPointers, aws.String(id)) + subnetInput := &ec2.DescribeSubnetsInput{} + if len(vpcID) > 0 { + subnetInput.Filters = append(subnetInput.Filters, ec2types.Filter{ + Name: ptr.To("vpc-id"), + Values: []string{vpcID}, + }) + } + if len(subnetIDs) > 0 { + subnetInput.SubnetIds = subnetIDs } - var lastError error - err = client.DescribeSubnetsPagesWithContext( - ctx, - &ec2.DescribeSubnetsInput{SubnetIds: idPointers}, - func(results *ec2.DescribeSubnetsOutput, lastPage bool) bool { - for _, subnet := range results.Subnets { - if subnet.SubnetId == nil { - continue - } - if subnet.SubnetArn == nil { - lastError = fmt.Errorf("%s has no ARN", *subnet.SubnetId) - return false - } - if subnet.VpcId == nil { - lastError = fmt.Errorf("%s has no VPC", *subnet.SubnetId) - return false - } - if subnet.AvailabilityZone == nil { - lastError = fmt.Errorf("%s has not availability zone", *subnet.SubnetId) - return false - } - - if subnetGroups.VPC == "" { - subnetGroups.VPC = *subnet.VpcId - vpcFromSubnet = *subnet.SubnetId - } else if *subnet.VpcId != subnetGroups.VPC { - lastError = fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC) - return false - } - metas[aws.StringValue(subnet.SubnetId)] = Subnet{ - ID: aws.StringValue(subnet.SubnetId), - ARN: aws.StringValue(subnet.SubnetArn), - Zone: &Zone{Name: aws.StringValue(subnet.AvailabilityZone)}, - CIDR: aws.StringValue(subnet.CidrBlock), - Public: false, - } - zoneNames = append(zoneNames, subnet.AvailabilityZone) + err := describeSubnets(ctx, client, subnetInput, func(subnets []ec2types.Subnet) error { + var vpcFromSubnet string + for _, subnet := range subnets { + if subnet.SubnetId == nil { + continue } - return !lastPage - }, - ) - if err == nil { - err = lastError - } + if len(ptr.Deref(subnet.SubnetArn, "")) == 0 { + return fmt.Errorf("%s has no ARN", *subnet.SubnetId) + } + if len(ptr.Deref(subnet.VpcId, "")) == 0 { + return fmt.Errorf("%s has no VPC", *subnet.SubnetId) + } + if len(ptr.Deref(subnet.AvailabilityZone, "")) == 0 { + return fmt.Errorf("%s has no availability zone", *subnet.SubnetId) + } + if subnetGroups.VPC == "" { + subnetGroups.VPC = *subnet.VpcId + vpcFromSubnet = *subnet.SubnetId + } else if *subnet.VpcId != subnetGroups.VPC { + return fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC) + } + + // At this point, we should be safe to dereference these fields. + metas[*subnet.SubnetId] = Subnet{ + ID: *subnet.SubnetId, + ARN: *subnet.SubnetArn, + Zone: &Zone{Name: *subnet.AvailabilityZone}, + CIDR: ptr.Deref(subnet.CidrBlock, ""), + Public: false, + Tags: FromAWSTags(subnet.Tags), + } + zoneNames = append(zoneNames, *subnet.AvailabilityZone) + } + return nil + }) if err != nil { - return subnetGroups, fmt.Errorf("describing subnets: %w", err) + return subnetGroups, err } - var routeTables []*ec2.RouteTable - err = client.DescribeRouteTablesPagesWithContext( - ctx, - &ec2.DescribeRouteTablesInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("vpc-id"), - Values: []*string{aws.String(subnetGroups.VPC)}, - }}, - }, - func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool { - routeTables = append(routeTables, results.RouteTables...) - return !lastPage - }, - ) + var routeTables []ec2types.RouteTable + err = describeRouteTables(ctx, client, &ec2.DescribeRouteTablesInput{ + Filters: []ec2types.Filter{{ + Name: ptr.To("vpc-id"), + Values: []string{subnetGroups.VPC}, + }}, + }, func(rTables []ec2types.RouteTable) error { + routeTables = append(routeTables, rTables...) + return nil + }) if err != nil { - return subnetGroups, fmt.Errorf("describing route tables: %w", err) + return subnetGroups, err } - azs, err := client.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames}) + azs, err := client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames}) if err != nil { return subnetGroups, fmt.Errorf("describing availability zones: %w", err) } @@ -140,6 +143,14 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ publicOnlySubnets := typesaws.IsPublicOnlySubnetsEnabled() + var ids []string + if len(vpcID) > 0 { + ids = metas.IDs() + } + if len(subnetIDs) > 0 { + ids = subnetIDs + } + for _, id := range ids { meta, ok := metas[id] if !ok { @@ -157,10 +168,10 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ return subnetGroups, fmt.Errorf("unable to read properties of zone name %s from the list %v: %w", zoneName, zoneNames, err) } zone := availabilityZones[zoneName] - meta.Zone.Type = aws.StringValue(zone.ZoneType) - meta.Zone.GroupName = aws.StringValue(zone.GroupName) + meta.Zone.Type = ptr.Deref(zone.ZoneType, "") + meta.Zone.GroupName = ptr.Deref(zone.GroupName, "") if availabilityZones[zoneName].ParentZoneName != nil { - meta.Zone.ParentZoneName = aws.StringValue(zone.ParentZoneName) + meta.Zone.ParentZoneName = ptr.Deref(zone.ParentZoneName, "") } // AWS Local Zones are grouped as Edge subnets @@ -189,12 +200,12 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ } // https://github.com/kubernetes/kubernetes/blob/9f036cd43d35a9c41d7ac4ca82398a6d0bef957b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L3376-L3419 -func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { - var subnetTable *ec2.RouteTable +func isSubnetPublic(rt []ec2types.RouteTable, subnetID string) (bool, error) { + var subnetTable *ec2types.RouteTable for _, table := range rt { for _, assoc := range table.Associations { - if aws.StringValue(assoc.SubnetId) == subnetID { - subnetTable = table + if ptr.Equal(assoc.SubnetId, &subnetID) { + subnetTable = &table break } } @@ -205,10 +216,10 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { // associated with the VPC's main routing table. for _, table := range rt { for _, assoc := range table.Associations { - if aws.BoolValue(assoc.Main) { + if ptr.Deref(assoc.Main, false) { logrus.Debugf("Assuming implicit use of main routing table %s for %s", - aws.StringValue(table.RouteTableId), subnetID) - subnetTable = table + ptr.Deref(table.RouteTableId, ""), subnetID) + subnetTable = &table break } } @@ -226,13 +237,56 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { // from the default in-subnet route which is called "local" // or other virtual gateway (starting with vgv) // or vpc peering connections (starting with pcx). - if strings.HasPrefix(aws.StringValue(route.GatewayId), "igw") { + if strings.HasPrefix(ptr.Deref(route.GatewayId, ""), "igw") { return true, nil } - if strings.HasPrefix(aws.StringValue(route.CarrierGatewayId), "cagw") { + if strings.HasPrefix(ptr.Deref(route.CarrierGatewayId, ""), "cagw") { return true, nil } } return false, nil } + +// describeSubnets retrieves metadata for subnets with given filters. +func describeSubnets(ctx context.Context, client *ec2.Client, input *ec2.DescribeSubnetsInput, fn func(subnets []ec2types.Subnet) error) error { + paginator := ec2.NewDescribeSubnetsPaginator(client, input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return fmt.Errorf("describing subnets: %w", err) + } + + // If the handler returns an error, we stop early to avoid extra API calls. + if err = fn(page.Subnets); err != nil { + return err + } + } + return nil +} + +// describeRouteTables retrieves metadata for route tables with given filters. +func describeRouteTables(ctx context.Context, client *ec2.Client, input *ec2.DescribeRouteTablesInput, fn func(subnets []ec2types.RouteTable) error) error { + paginator := ec2.NewDescribeRouteTablesPaginator(client, input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return fmt.Errorf("describing route tables: %w", err) + } + + // If the handler returns an error, we stop early to avoid extra API calls. + if err = fn(page.RouteTables); err != nil { + return err + } + } + return nil +} + +// mergeSubnets merged two or more Subnets into a single one for convenience. +func mergeSubnets(groups ...Subnets) Subnets { + subnets := make(Subnets) + for _, group := range groups { + maps.Copy(subnets, group) + } + return subnets +} diff --git a/pkg/asset/installconfig/aws/tags.go b/pkg/asset/installconfig/aws/tags.go new file mode 100644 index 0000000000..38f314fa84 --- /dev/null +++ b/pkg/asset/installconfig/aws/tags.go @@ -0,0 +1,44 @@ +package aws + +import ( + "strings" + + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/utils/ptr" +) + +const ( + // TagNameKubernetesClusterPrefix is the tag name prefix used by CCM + // to differentiate multiple logically independent clusters running in the same AZ. + TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/" + + // TagNameKubernetesUnmanaged is the tag name to indicate that a resource is unmanaged + // by the cluster and should be ignored by CCM. For example, kubernetes.io/cluster/unmanaged=true. + TagNameKubernetesUnmanaged = TagNameKubernetesClusterPrefix + "unmanaged" +) + +// Tags represents AWS resource tags as a map. +// This helps avoid iterating over the tag list for every lookup. +type Tags map[string]string + +// FromAWSTags converts a list of AWS tags into a map. +func FromAWSTags(awsTags []types.Tag) Tags { + tags := make(Tags, len(awsTags)) + for _, tag := range awsTags { + key, value := ptr.Deref(tag.Key, ""), ptr.Deref(tag.Value, "") + if len(key) > 0 { + tags[key] = value + } + } + return tags +} + +// HasTagKeyPrefix returns true if there is a tag with a given key prefix. +func (t Tags) HasTagKeyPrefix(prefix string) bool { + for key := range t { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go index fc7a4e51e3..60ef57aae6 100644 --- a/pkg/asset/installconfig/aws/validation.go +++ b/pkg/asset/installconfig/aws/validation.go @@ -50,7 +50,7 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig) allErrs = append(allErrs, validateAMI(ctx, meta, config)...) allErrs = append(allErrs, validatePublicIpv4Pool(ctx, meta, field.NewPath("platform", "aws", "publicIpv4PoolId"), config)...) - allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config.Platform.AWS, config.Networking, config.Publish)...) + allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config)...) if awstypes.IsPublicOnlySubnetsEnabled() { logrus.Warnln("Public-only subnets install. Please be warned this is not supported") @@ -98,8 +98,9 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig) return allErrs.ToAggregate() } -func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList { +func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList { allErrs := field.ErrorList{} + platform := config.Platform.AWS allErrs = append(allErrs, validateServiceEndpoints(fldPath.Child("serviceEndpoints"), platform.Region, platform.ServiceEndpoints)...) @@ -109,7 +110,7 @@ func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, } if len(platform.VPC.Subnets) > 0 { - allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), platform.VPC.Subnets, networking, publish)...) + allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), config)...) } if platform.DefaultMachinePlatform != nil { allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("defaultMachinePlatform"), platform, platform.DefaultMachinePlatform, controlPlaneReq, "", "")...) @@ -218,67 +219,139 @@ func validatePublicIpv4Pool(ctx context.Context, meta *Metadata, fldPath *field. return nil } -func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, subnets []awstypes.Subnet, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList { - allErrs := field.ErrorList{} - privateSubnets, err := meta.PrivateSubnets(ctx) - if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) +// subnetData holds a subnet information collected from install config and AWS API for validations. +type subnetData struct { + // The subnet index in the install config. + Idx int + // The subnet assigned roles in the install config. + Roles []awstypes.SubnetRole + // The subnet metadata from AWS API. + Subnet +} + +// subnetDataGroups is a collection of subnet information +// grouped by subnet type (i.e. public, private, and edge) and indexed by subnetIDs for validations. +type subnetDataGroups struct { + Public map[string]subnetData + Private map[string]subnetData + Edge map[string]subnetData + // A convenient alias that contains all information for all subnets. + All map[string]subnetData +} + +// Converts subnetGroups (i.e. provided subnets) to subnetDataGroups to include additional information +// from the install-config such as index and roles for validations. +func (sdg *subnetDataGroups) From(ctx context.Context, meta *Metadata, providedSubnets []awstypes.Subnet) error { + if sdg.Private == nil { + sdg.Private = make(map[string]subnetData) } - privateSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := privateSubnets[string(subnet.ID)]; ok { - privateSubnetsIdx[string(subnet.ID)] = idx - } + if sdg.Public == nil { + sdg.Public = make(map[string]subnetData) } - if len(privateSubnets) == 0 { - allErrs = append(allErrs, field.Invalid(fldPath, subnets, "No private subnets found")) + if sdg.Edge == nil { + sdg.Edge = make(map[string]subnetData) + } + if sdg.All == nil { + sdg.All = make(map[string]subnetData) } - publicSubnets, err := meta.PublicSubnets(ctx) + subnets, err := meta.Subnets(ctx) if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) + return err } - if publish == types.InternalPublishingStrategy && len(publicSubnets) > 0 { - logrus.Warnf("Public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy) - } - publicSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := publicSubnets[string(subnet.ID)]; ok { - publicSubnetsIdx[string(subnet.ID)] = idx + + for idx, subnet := range providedSubnets { + var subnetDataGroup map[string]subnetData + var subnetMeta Subnet + + if awsSubnet, ok := subnets.Private[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Private + subnetMeta = awsSubnet } + + if awsSubnet, ok := subnets.Public[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Public + subnetMeta = awsSubnet + } + + if awsSubnet, ok := subnets.Edge[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Edge + subnetMeta = awsSubnet + } + + if subnetDataGroup == nil { + // Should not occur but safe against panics + continue + } + + subnetData := subnetData{ + Subnet: subnetMeta, + Idx: idx, + Roles: subnet.Roles, + } + subnetDataGroup[string(subnet.ID)] = subnetData + sdg.All[string(subnet.ID)] = subnetData } - if len(publicSubnets) == 0 && awstypes.IsPublicOnlySubnetsEnabled() { + + return nil +} + +// validateSubnets ensures BYO subnets are valid. +func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList { + allErrs := field.ErrorList{} + networking := config.Networking + providedSubnets := config.AWS.VPC.Subnets + publish := config.Publish + + subnetDataGroups := subnetDataGroups{} + if err := subnetDataGroups.From(ctx, meta, providedSubnets); err != nil { + return append(allErrs, field.Invalid(fldPath, providedSubnets, err.Error())) + } + + publicOnlySubnet := awstypes.IsPublicOnlySubnetsEnabled() + + if publicOnlySubnet && len(subnetDataGroups.Public) == 0 { allErrs = append(allErrs, field.Required(fldPath, "public subnets are required for a public-only subnets cluster")) } - edgeSubnets, err := meta.EdgeSubnets(ctx) - if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) + if !publicOnlySubnet && len(subnetDataGroups.Private) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, "no private subnets found")) } - edgeSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := edgeSubnets[string(subnet.ID)]; ok { - edgeSubnetsIdx[string(subnet.ID)] = idx + + if publish == types.InternalPublishingStrategy && len(subnetDataGroups.Public) > 0 { + logrus.Warnf("public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy) + } + + subnetsWithRole := make(map[awstypes.SubnetRoleType][]subnetData) + for _, subnet := range providedSubnets { + for _, role := range subnet.Roles { + subnetsWithRole[role.Type] = append(subnetsWithRole[role.Type], subnetDataGroups.All[string(subnet.ID)]) } } - allErrs = append(allErrs, validateSubnetCIDR(fldPath, privateSubnets, privateSubnetsIdx, networking.MachineNetwork)...) - allErrs = append(allErrs, validateSubnetCIDR(fldPath, publicSubnets, publicSubnetsIdx, networking.MachineNetwork)...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, privateSubnets, privateSubnetsIdx, "private")...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, publicSubnets, publicSubnetsIdx, "public")...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, edgeSubnets, edgeSubnetsIdx, "edge")...) + allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Private, networking.MachineNetwork)...) + allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Public, networking.MachineNetwork)...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Private, "private")...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Public, "public")...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Edge, "edge")...) + + if len(subnetsWithRole) > 0 { + allErrs = append(allErrs, validateSubnetRoles(fldPath, subnetsWithRole, subnetDataGroups, config)...) + } else { + allErrs = append(allErrs, validateUntaggedSubnets(ctx, fldPath, meta, subnetDataGroups)...) + } privateZones := sets.New[string]() publicZones := sets.New[string]() - for _, subnet := range privateSubnets { + for _, subnet := range subnetDataGroups.Private { privateZones.Insert(subnet.Zone.Name) } - for _, subnet := range publicSubnets { + for _, subnet := range subnetDataGroups.Public { publicZones.Insert(subnet.Zone.Name) } if publish == types.ExternalPublishingStrategy && !publicZones.IsSuperset(privateZones) { errMsg := fmt.Sprintf("No public subnet provided for zones %s", sets.List(privateZones.Difference(publicZones))) - allErrs = append(allErrs, field.Invalid(fldPath, subnets, errMsg)) + allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, errMsg)) } return allErrs @@ -436,11 +509,11 @@ func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *fiel return allErrs } -func validateSubnetCIDR(fldPath *field.Path, subnets Subnets, idxMap map[string]int, networks []types.MachineNetworkEntry) field.ErrorList { +func validateSubnetCIDR(fldPath *field.Path, subnetDataGroup map[string]subnetData, networks []types.MachineNetworkEntry) field.ErrorList { allErrs := field.ErrorList{} - for id, v := range subnets { - fp := fldPath.Index(idxMap[id]) - cidr, _, err := net.ParseCIDR(v.CIDR) + for id, subnetData := range subnetDataGroup { + fp := fldPath.Index(subnetData.Idx) + cidr, _, err := net.ParseCIDR(subnetData.CIDR) if err != nil { allErrs = append(allErrs, field.Invalid(fp, id, err.Error())) continue @@ -459,27 +532,215 @@ func validateMachineNetworksContainIP(fldPath *field.Path, networks []types.Mach return field.ErrorList{field.Invalid(fldPath, subnetName, fmt.Sprintf("subnet's CIDR range start %s is outside of the specified machine networks", ip))} } -func validateDuplicateSubnetZones(fldPath *field.Path, subnets Subnets, idxMap map[string]int, typ string) field.ErrorList { - var keys []string - for id := range subnets { - keys = append(keys, id) +func validateDuplicateSubnetZones(fldPath *field.Path, subnetDataGroup map[string]subnetData, typ string) field.ErrorList { + subnetIDs := make([]string, 0) + for id := range subnetDataGroup { + subnetIDs = append(subnetIDs, id) } - sort.Strings(keys) + sort.Strings(subnetIDs) allErrs := field.ErrorList{} zones := map[string]string{} - for _, id := range keys { - subnet := subnets[id] - if conflictingSubnet, ok := zones[subnet.Zone.Name]; ok { - errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnet.Zone.Name) - allErrs = append(allErrs, field.Invalid(fldPath.Index(idxMap[id]), id, errMsg)) + for _, id := range subnetIDs { + subnetData := subnetDataGroup[id] + if conflictingSubnet, ok := zones[subnetData.Zone.Name]; ok { + errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnetData.Zone.Name) + allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), id, errMsg)) } else { - zones[subnet.Zone.Name] = id + zones[subnetData.Zone.Name] = id } } return allErrs } +// validateSubnetRoles ensures BYO subnets have valid roles assigned if roles are provided. +func validateSubnetRoles(fldPath *field.Path, subnetsWithRole map[awstypes.SubnetRoleType][]subnetData, subnetDataGroups subnetDataGroups, config *types.InstallConfig) field.ErrorList { + allErrs := field.ErrorList{} + + // BootstrapNode subnets must be assigned to public subnets + // in external cluster. + for _, bstrSubnet := range subnetsWithRole[awstypes.BootstrapNodeSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[bstrSubnet.ID]; ok { + continue + } + if config.Publish == types.ExternalPublishingStrategy && !bstrSubnet.Public { + allErrs = append(allErrs, field.Invalid(fldPath.Index(bstrSubnet.Idx), bstrSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", bstrSubnet.ID, awstypes.BootstrapNodeSubnetRole))) + } + } + + // ClusterNode subnets must be assigned to private subnets + // unless cluster is public-only. + for _, cnSubnet := range subnetsWithRole[awstypes.ClusterNodeSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[cnSubnet.ID]; ok { + continue + } + if cnSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() { + allErrs = append(allErrs, field.Invalid(fldPath.Index(cnSubnet.Idx), cnSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", cnSubnet.ID, awstypes.ClusterNodeSubnetRole))) + } + } + + // Type of ControlPlaneLB subnets must match its scope: + // - ControlPlaneInternalLB subnets must be private + // - ControlPlaneExternalLB subnets must be public. + // Private cluster must not have ControlPlaneExternalLB subnets (i.e. statically validated in pkg/types/aws/validation/platform.go). + for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneInternalLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok { + continue + } + if ctrlPSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() { + allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", ctrlPSubnet.ID, awstypes.ControlPlaneInternalLBSubnetRole))) + } + } + for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneExternalLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok { + continue + } + if !ctrlPSubnet.Public { + allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", ctrlPSubnet.ID, awstypes.ControlPlaneExternalLBSubnetRole))) + } + } + + // Type of IngressControllerLB subnets must match cluster scope: + // - In public cluster, only public IngressControllerLB subnets is allowed. + // - In private cluster, only private IngressControllerLB subnets is allowed. + for _, ingressSubnet := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ingressSubnet.ID]; ok { + continue + } + + if ingressSubnet.Public != config.PublicIngress() { + subnetType := "private" + if ingressSubnet.Public { + subnetType = "public" + } + allErrs = append(allErrs, field.Invalid(fldPath.Index(ingressSubnet.Idx), ingressSubnet.ID, + fmt.Sprintf("subnet %s has role %s and is %s, which is not allowed when publish is set to %s", ingressSubnet.ID, awstypes.IngressControllerLBSubnetRole, subnetType, config.Publish))) + } + } + + // IngressControllerLB subnets must be in different AZs as required by AWS CCM. + ingressZones := make(map[string]string) + for _, subnetData := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] { + if conflictingSubnet, ok := ingressZones[subnetData.Zone.Name]; ok { + allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), subnetData.ID, + fmt.Sprintf("subnets %s and %s have role %s and are both in zone %s", conflictingSubnet, subnetData.ID, awstypes.IngressControllerLBSubnetRole, subnetData.Zone.Name))) + } else { + ingressZones[subnetData.Zone.Name] = subnetData.ID + } + } + + // AZs of LB subnets match AZs of ClusterNode subnets. + lbRoles := []awstypes.SubnetRoleType{ + awstypes.ControlPlaneInternalLBSubnetRole, + awstypes.IngressControllerLBSubnetRole, + } + if config.Publish == types.ExternalPublishingStrategy { + lbRoles = append(lbRoles, awstypes.ControlPlaneExternalLBSubnetRole) + } + for _, role := range lbRoles { + allErrs = append(allErrs, validateLBSubnetAZMatchClusterNodeAZ(fldPath, subnetDataGroups, role, subnetsWithRole[role], subnetsWithRole[awstypes.ClusterNodeSubnetRole])...) + } + + // EdgeNode subnets must be subnets in Local or Wavelength Zones. + for _, edgeSubnet := range subnetsWithRole[awstypes.EdgeNodeSubnetRole] { + if _, ok := subnetDataGroups.Edge[edgeSubnet.ID]; !ok { + allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is not in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole))) + } + } + + // Subnets that are in Local or Wavelength Zones must only have EdgeNode role. + for _, edgeSubnet := range subnetDataGroups.Edge { + for _, role := range edgeSubnet.Roles { + if role.Type != awstypes.EdgeNodeSubnetRole { + allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID, + fmt.Sprintf("subnet %s must only be assigned role %s since it is in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole))) + break + } + } + } + + return allErrs +} + +// validateUntaggedSubnets ensures there are no additional untagged subnets in the BYO VPC. +// An untagged subnet is a subnet without tag kubernetes.io/cluster/. +// Untagged subnets may be selected by the CCM, leading to various bugs, RFEs, and support cases. See: +// - https://issues.redhat.com/browse/OCPBUGS-17432. +// - https://issues.redhat.com/browse/RFE-2816. +func validateUntaggedSubnets(ctx context.Context, fldPath *field.Path, meta *Metadata, subnetDataGroups subnetDataGroups) field.ErrorList { + allErrs := field.ErrorList{} + + vpcSubnets, err := meta.VPCSubnets(ctx) + if err != nil { + return append(allErrs, field.Invalid(fldPath, meta.ProvidedSubnets, err.Error())) + } + + untaggedSubnetIDs := make([]string, 0) + for _, subnet := range mergeSubnets(vpcSubnets.Public, vpcSubnets.Private, vpcSubnets.Edge) { + // We only check other subnets in the VPC that are not provided in the install-config. + if _, ok := subnetDataGroups.All[subnet.ID]; !ok && !subnet.Tags.HasTagKeyPrefix(TagNameKubernetesClusterPrefix) { + untaggedSubnetIDs = append(untaggedSubnetIDs, subnet.ID) + } + } + sort.Strings(untaggedSubnetIDs) + + if len(untaggedSubnetIDs) > 0 { + errMsg := fmt.Sprintf("additional subnets %v without tag prefix %s are found in vpc %s of provided subnets. %s", untaggedSubnetIDs, TagNameKubernetesClusterPrefix, vpcSubnets.VPC, + fmt.Sprintf("Please add a tag %s to those subnets to exclude them from cluster installation or explicitly assign roles in the install-config to provided subnets", TagNameKubernetesUnmanaged)) + allErrs = append(allErrs, field.Forbidden(fldPath, errMsg)) + } + + return allErrs +} + +// validateLBSubnetAZMatchClusterNodeAZ ensures AZs of LB subnets match AZs of ClusterNode subnets. +// AWS load balancers will NOT register a node located in an AZ that is not enabled for the load balancer. +func validateLBSubnetAZMatchClusterNodeAZ(fldPath *field.Path, subnetDataGroups subnetDataGroups, lbType awstypes.SubnetRoleType, lbSubnets []subnetData, clusterNodeSubnets []subnetData) field.ErrorList { + allErrs := field.ErrorList{} + + lbZoneSet := sets.New[string]() + for _, subnet := range lbSubnets { + // We validate edge subnets in another place. + if _, ok := subnetDataGroups.Edge[subnet.ID]; ok { + continue + } + lbZoneSet.Insert(subnet.Zone.Name) + } + + nodeZoneSet := sets.New[string]() + for _, subnet := range clusterNodeSubnets { + // We validate edge subnets in another place. + if _, ok := subnetDataGroups.Edge[subnet.ID]; ok { + continue + } + nodeZoneSet.Insert(subnet.Zone.Name) + } + + // If the nodes use an AZ that is not in load balancer enabled AZs, + // the router pod might be scheduled to nodes that the load balancer cannot reach. + if diffSet := nodeZoneSet.Difference(lbZoneSet); diffSet.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are not enabled for %s load balancers, nodes in those zones are unreachable", sets.List(diffSet), lbType))) + } + + // If the load balancer includes an AZ that is not in node AZs, + // there will be no nodes in that AZ for the load balancer to register (i.e. not in use) + if diffSet := lbZoneSet.Difference(nodeZoneSet); diffSet.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are enabled for %s load balancers, but are not used by any nodes", sets.List(diffSet), lbType))) + } + + return allErrs +} + func validateServiceEndpoints(fldPath *field.Path, region string, services []awstypes.ServiceEndpoint) field.ErrorList { allErrs := field.ErrorList{} // Validate the endpoint overrides for all provided services. diff --git a/pkg/asset/installconfig/aws/validation_test.go b/pkg/asset/installconfig/aws/validation_test.go index f15ff17ef3..43172603cb 100644 --- a/pkg/asset/installconfig/aws/validation_test.go +++ b/pkg/asset/installconfig/aws/validation_test.go @@ -457,7 +457,7 @@ func TestValidate(t *testing.T) { }(), availZones: validAvailZones(), publicSubnets: validPublicSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, availRegions: validAvailRegions(), }, { name: "invalid no public subnets", @@ -681,7 +681,7 @@ func TestValidate(t *testing.T) { privateSubnets: Subnets{}, publicSubnets: Subnets{}, edgeSubnets: validEdgeSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, availRegions: validAvailRegions(), }, { name: "invalid no subnet for control plane zones", @@ -953,11 +953,21 @@ func TestValidate(t *testing.T) { meta := &Metadata{ availabilityZones: test.availZones, availableRegions: test.availRegions, - privateSubnets: test.privateSubnets, - publicSubnets: test.publicSubnets, - edgeSubnets: test.edgeSubnets, - instanceTypes: test.instanceTypes, - Subnets: test.installConfig.Platform.AWS.VPC.Subnets, + subnets: SubnetGroups{ + Private: test.privateSubnets, + Public: test.publicSubnets, + Edge: test.edgeSubnets, + VPC: "valid-vpc", + }, + vpcSubnets: SubnetGroups{ + Private: test.privateSubnets, + Public: test.publicSubnets, + Edge: test.edgeSubnets, + VPC: "valid-vpc", + }, + vpc: "valid-vpc", + instanceTypes: test.instanceTypes, + ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets, } if test.proxy != "" { os.Setenv("HTTP_PROXY", test.proxy) @@ -1090,12 +1100,20 @@ func TestValidateForProvisioning(t *testing.T) { meta := &Metadata{ availabilityZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - instanceTypes: validInstanceTypes(), - Region: editedInstallConfig.AWS.Region, - vpc: "valid-private-subnet-a", - Subnets: editedInstallConfig.Platform.AWS.VPC.Subnets, + subnets: SubnetGroups{ + Private: validPrivateSubnets(), + Public: validPublicSubnets(), + VPC: "valid-vpc", + }, + vpcSubnets: SubnetGroups{ + Private: validPrivateSubnets(), + Public: validPublicSubnets(), + VPC: "valid-vpc", + }, + instanceTypes: validInstanceTypes(), + Region: editedInstallConfig.AWS.Region, + vpc: "valid-private-subnet-a", + ProvidedSubnets: editedInstallConfig.Platform.AWS.VPC.Subnets, } err := ValidateForProvisioning(route53Client, editedInstallConfig, meta) From d2e91e9633d39bd4ee06d76bd68db89acf4b5483 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Tue, 25 Mar 2025 17:51:52 -0700 Subject: [PATCH 2/2] tests: add unit tests for the field vpc.subnets with mock AWS API Test cases are added to cover the new BYO subnet validations mentioned in the previous commit. Other related changes: - The test file is also refactored to deduplicate, and standardize the test constructs. - Http responses are mocked with httpmock instead of hacking with e2e.local domain. - Wrap errors in a more user-friendly descriptions when validating service endpoint accessibility. --- .../installconfig/aws/permissions_test.go | 172 +- pkg/asset/installconfig/aws/validation.go | 14 +- .../installconfig/aws/validation_test.go | 2564 ++++++++++------- 3 files changed, 1713 insertions(+), 1037 deletions(-) diff --git a/pkg/asset/installconfig/aws/permissions_test.go b/pkg/asset/installconfig/aws/permissions_test.go index 2068af8c92..68f5a2fa09 100644 --- a/pkg/asset/installconfig/aws/permissions_test.go +++ b/pkg/asset/installconfig/aws/permissions_test.go @@ -5,7 +5,9 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "github.com/openshift/installer/pkg/ipnet" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/aws" ) @@ -21,6 +23,58 @@ func basicInstallConfig() types.InstallConfig { } } +// validBYOSubnetsInstallConfig returns a valid install config for BYO subnets use case. +// Test cases can unset fields if necessary. +func validBYOSubnetsInstallConfig() *types.InstallConfig { + return &types.InstallConfig{ + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{ + {CIDR: *ipnet.MustParseCIDR(validCIDR)}, + }, + }, + BaseDomain: validDomainName, + Publish: types.ExternalPublishingStrategy, + Platform: types.Platform{ + AWS: &aws.Platform{ + Region: "us-east-1", + VPC: aws.VPC{ + Subnets: []aws.Subnet{ + {ID: "subnet-valid-private-a"}, + {ID: "subnet-valid-private-b"}, + {ID: "subnet-valid-private-c"}, + {ID: "subnet-valid-public-a"}, + {ID: "subnet-valid-public-b"}, + {ID: "subnet-valid-public-c"}, + }, + }, + HostedZone: validHostedZoneName, + }, + }, + ControlPlane: &types.MachinePool{ + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "c"}, + }, + }, + }, + Compute: []types.MachinePool{{ + Name: types.MachinePoolComputeRoleName, + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "c"}, + }, + }, + }}, + ObjectMeta: metav1.ObjectMeta{ + Name: metaName, + }, + } +} + func TestIncludesCreateInstanceRole(t *testing.T) { t.Run("Should be true when", func(t *testing.T) { t.Run("no machine types specified", func(t *testing.T) { @@ -310,28 +364,28 @@ func TestIAMRolePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create and delete shared IAM role permissions", func(t *testing.T) { t.Run("when role specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when role specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) @@ -340,7 +394,7 @@ func TestIAMRolePermissions(t *testing.T) { }) t.Run("create IAM role permissions", func(t *testing.T) { t.Run("when no existing roles and instance profiles are specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) @@ -350,7 +404,7 @@ func TestIAMRolePermissions(t *testing.T) { t.Run("Should not include create IAM role permissions", func(t *testing.T) { t.Run("when role specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMRole: "custom-default-role", } @@ -359,7 +413,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when role specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role" ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role" requiredPerms := RequiredPermissionGroups(ic) @@ -367,7 +421,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMProfile: "custom-default-profile", } @@ -376,7 +430,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) @@ -390,14 +444,14 @@ func TestIAMProfilePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create and delete shared instance profile permissions", func(t *testing.T) { t.Run("when instance profile specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) }) t.Run("when instance profile specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) @@ -406,7 +460,7 @@ func TestIAMProfilePermissions(t *testing.T) { }) t.Run("create instance profile permissions", func(t *testing.T) { t.Run("when no existing instance profiles are specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) @@ -416,7 +470,7 @@ func TestIAMProfilePermissions(t *testing.T) { t.Run("Should not include create instance profile permissions", func(t *testing.T) { t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMProfile: "custom-default-profile", } @@ -425,7 +479,7 @@ func TestIAMProfilePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) }) t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) @@ -527,7 +581,7 @@ func TestIncludesKMSEncryptionKeys(t *testing.T) { func TestKMSKeyPermissions(t *testing.T) { t.Run("Should include KMS key permissions", func(t *testing.T) { t.Run("when KMS key specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.EC2RootVolume = aws.EC2RootVolume{ KMSKeyARN: "custom-master-key", } @@ -535,7 +589,7 @@ func TestKMSKeyPermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when KMS key specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.EC2RootVolume = aws.EC2RootVolume{ KMSKeyARN: "custom-worker-key", } @@ -543,7 +597,7 @@ func TestKMSKeyPermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when KMS key specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ EC2RootVolume: aws.EC2RootVolume{ KMSKeyARN: "custom-default-key", @@ -556,14 +610,14 @@ func TestKMSKeyPermissions(t *testing.T) { t.Run("Should not include KMS key permissions", func(t *testing.T) { t.Run("when no machine types specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane = nil ic.Compute = nil requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when no KMS keys specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys) @@ -575,14 +629,14 @@ func TestVPCPermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create network permissions when VPC not specified", func(t *testing.T) { t.Run("for standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateNetworking) }) t.Run("for secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" @@ -591,32 +645,32 @@ func TestVPCPermissions(t *testing.T) { }) }) t.Run("delete network permissions when VPC not specified for standard region", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteNetworking) }) t.Run("delete shared network permissions when VPC specified for standard region", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteSharedNetworking) }) }) t.Run("Should not include", func(t *testing.T) { t.Run("create network permissions when VPC specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCreateNetworking) }) t.Run("delete network permissions", func(t *testing.T) { t.Run("when VPC specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteNetworking) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteNetworking) @@ -624,14 +678,14 @@ func TestVPCPermissions(t *testing.T) { }) t.Run("delete shared network permissions", func(t *testing.T) { t.Run("when VPC not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking) @@ -643,13 +697,13 @@ func TestVPCPermissions(t *testing.T) { func TestPrivateZonePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create hosted zone permissions when PHZ not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateHostedZone) }) t.Run("delete hosted zone permissions when PHZ not specified on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteHostedZone) @@ -657,18 +711,18 @@ func TestPrivateZonePermissions(t *testing.T) { }) t.Run("Should not include", func(t *testing.T) { t.Run("create hosted zone permissions when PHZ specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCreateHostedZone) }) t.Run("delete hosted zone permissions", func(t *testing.T) { t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone) }) t.Run("when PHZ specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone) }) @@ -678,13 +732,13 @@ func TestPrivateZonePermissions(t *testing.T) { func TestPublicIPv4PoolPermissions(t *testing.T) { t.Run("Should include IPv4Pool permissions when IPv4 pool specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.PublicIpv4Pool = "custom-ipv4-pool" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionPublicIpv4Pool) }) t.Run("Should not include IPv4Pool permissions when IPv4 pool not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionPublicIpv4Pool) }) @@ -694,25 +748,25 @@ func TestBasePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("base create permissions", func(t *testing.T) { t.Run("on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateBase) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateBase) }) }) t.Run("base delete permissions on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteBase) }) }) t.Run("Should not include base delete permissions on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteBase) @@ -721,12 +775,12 @@ func TestBasePermissions(t *testing.T) { func TestDeleteIgnitionPermissions(t *testing.T) { t.Run("Should include delete ignition permissions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteIgnitionObjects) }) t.Run("Should not include delete ignition permission when specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.BestEffortDeleteIgnition = true requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteIgnitionObjects) @@ -737,7 +791,7 @@ func TestIncludesInstanceType(t *testing.T) { const instanceType = "m7a.2xlarge" t.Run("Should be true when instance type specified for", func(t *testing.T) { t.Run("defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ InstanceType: instanceType, } @@ -745,20 +799,20 @@ func TestIncludesInstanceType(t *testing.T) { assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) t.Run("controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.InstanceType = instanceType requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) t.Run("compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.InstanceType = instanceType requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) }) t.Run("Should be false when instance type is not set", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() assert.NotContains(t, RequiredPermissionGroups(ic), PermissionValidateInstanceType) }) } @@ -766,7 +820,7 @@ func TestIncludesInstanceType(t *testing.T) { func TestIncludesZones(t *testing.T) { t.Run("Should be true when", func(t *testing.T) { t.Run("zones specified in defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} @@ -777,21 +831,21 @@ func TestIncludesZones(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("zones specified in controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("zones specified in compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("subnets specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} requiredPerms := RequiredPermissionGroups(ic) @@ -799,7 +853,7 @@ func TestIncludesZones(t *testing.T) { }) }) t.Run("Should be false when neither zones nor subnets specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = []aws.Subnet{} ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} @@ -810,13 +864,13 @@ func TestIncludesZones(t *testing.T) { func TestIncludesAssumeRole(t *testing.T) { t.Run("Should be true when IAM role specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZoneRole = "custom-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionAssumeRole) }) t.Run("Should be false when IAM role not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionAssumeRole) }) @@ -824,7 +878,7 @@ func TestIncludesAssumeRole(t *testing.T) { func TestIncludesWavelengthZones(t *testing.T) { t.Run("Should be true when edge compute specified with WL zones", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -838,7 +892,7 @@ func TestIncludesWavelengthZones(t *testing.T) { }) t.Run("Should be false when", func(t *testing.T) { t.Run("edge compute specified without WL zones", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -851,7 +905,7 @@ func TestIncludesWavelengthZones(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionCarrierGateway) }) t.Run("edge compute not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCarrierGateway) }) @@ -861,7 +915,7 @@ func TestIncludesWavelengthZones(t *testing.T) { func TestIncludesEdgeDefaultInstance(t *testing.T) { t.Run("Should be true when at least one edge compute pool specified", func(t *testing.T) { t.Run("without platform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", }) @@ -878,7 +932,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { assert.Contains(t, requiredPerms, PermissionEdgeDefaultInstance) }) t.Run("without instance type", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -902,7 +956,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { }) t.Run("Should be false when", func(t *testing.T) { t.Run("edge compute specified with instance type", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -916,7 +970,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance) }) t.Run("edge compute not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance) }) diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go index 60ef57aae6..cee76e9964 100644 --- a/pkg/asset/installconfig/aws/validation.go +++ b/pkg/asset/installconfig/aws/validation.go @@ -788,17 +788,13 @@ func validateZoneLocal(ctx context.Context, meta *Metadata, fldPath *field.Path, } func validateEndpointAccessibility(endpointURL string) error { - // For each provided service endpoint, verify we can resolve and connect with net.Dial. - // Ignore e2e.local from unit tests. - if endpointURL == "e2e.local" { - return nil + if _, err := url.Parse(endpointURL); err != nil { + return fmt.Errorf("failed to parse service endpoint url: %w", err) } - _, err := url.Parse(endpointURL) - if err != nil { - return err + if _, err := http.Head(endpointURL); err != nil { //nolint:gosec + return fmt.Errorf("failed to connect to service endpoint url: %w", err) } - _, err = http.Head(endpointURL) - return err + return nil } var requiredServices = []string{ diff --git a/pkg/asset/installconfig/aws/validation_test.go b/pkg/asset/installconfig/aws/validation_test.go index 43172603cb..5e0622f1fe 100644 --- a/pkg/asset/installconfig/aws/validation_test.go +++ b/pkg/asset/installconfig/aws/validation_test.go @@ -3,12 +3,15 @@ package aws import ( "context" "fmt" + "net/http" "os" "sort" + "strings" "testing" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/route53" + "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,963 +25,1070 @@ import ( ) var ( - validCIDR = "10.0.0.0/16" - validRegion = "us-east-1" + metaName = "ClusterMetaName" + validCIDR = "10.0.0.0/16" + validVPCID = "vpc-valid-id" + validCallerRef = "valid-caller-reference" validDSId = "valid-delegation-set-id" - validNameServers = []string{"valid-name-server"} - validHostedZoneName = "valid-private-subnet-a" + validHostedZoneName = "valid-hosted-zone" invalidHostedZoneName = "invalid-hosted-zone" - validDomainName = "valid-base-domain" - invalidBaseDomain = "invalid-base-domain" - metaName = "ClusterMetaName" + validNameServers = []string{"valid-name-server"} - publishInternal = func(ic *types.InstallConfig) { ic.Publish = types.InternalPublishingStrategy } - clearHostedZone = func(ic *types.InstallConfig) { ic.AWS.HostedZone = "" } - invalidateHostedZone = func(ic *types.InstallConfig) { ic.AWS.HostedZone = invalidHostedZoneName } - invalidateBaseDomain = func(ic *types.InstallConfig) { ic.BaseDomain = invalidBaseDomain } - clearBaseDomain = func(ic *types.InstallConfig) { ic.BaseDomain = "" } - invalidateRegion = func(ic *types.InstallConfig) { ic.AWS.Region = "us-east4" } + validDomainName = "valid-base-domain" + invalidBaseDomain = "invalid-base-domain" + + // Convert IDs to Subnet type. + subnetsFromIDs = func(ids []string) []aws.Subnet { + subnets := make([]aws.Subnet, len(ids)) + for idx, id := range ids { + subnets[idx] = aws.Subnet{ID: aws.AWSSubnetID(id)} + } + // We sort the by ID to ensure a predictable error output in tests + sort.Slice(subnets, func(i, j int) bool { + return subnets[i].ID < subnets[j].ID + }) + return subnets + } + + // Remove http or https scheme from a URL if any. + trimURLScheme = func(url string) string { + if str, found := strings.CutPrefix(url, "https://"); found { + return str + } + if str, found := strings.CutPrefix(url, "http://"); found { + return str + } + return url + } ) -type editFunctions []func(ic *types.InstallConfig) - -func validInstallConfig() *types.InstallConfig { - return &types.InstallConfig{ - Networking: &types.Networking{ - MachineNetwork: []types.MachineNetworkEntry{ - {CIDR: *ipnet.MustParseCIDR(validCIDR)}, - }, - }, - BaseDomain: validDomainName, - Publish: types.ExternalPublishingStrategy, - Platform: types.Platform{ - AWS: &aws.Platform{ - Region: "us-east-1", - VPC: aws.VPC{ - Subnets: []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - {ID: "valid-public-subnet-a"}, - {ID: "valid-public-subnet-b"}, - {ID: "valid-public-subnet-c"}, - }, - }, - HostedZone: validHostedZoneName, - }, - }, - ControlPlane: &types.MachinePool{ - Architecture: types.ArchitectureAMD64, - Replicas: ptr.To[int64](3), - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "c"}, - }, - }, - }, - Compute: []types.MachinePool{{ - Name: types.MachinePoolComputeRoleName, - Architecture: types.ArchitectureAMD64, - Replicas: ptr.To[int64](3), - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "c"}, - }, - }, - }}, - ObjectMeta: metav1.ObjectMeta{ - Name: metaName, - }, - } -} - -// validInstallConfigEdgeSubnets returns install-config for edge compute pool -// for existing VPC (subnets). -func validInstallConfigEdgeSubnets() *types.InstallConfig { - ic := validInstallConfig() - edgeSubnets := validEdgeSubnets() - for subnet := range edgeSubnets { - ic.Platform.AWS.VPC.Subnets = append(ic.Platform.AWS.VPC.Subnets, aws.Subnet{ID: aws.AWSSubnetID(subnet)}) - } - ic.Compute = append(ic.Compute, types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{}, - }, - }) - return ic -} - -func validAvailZones() []string { - return []string{"a", "b", "c"} -} - -func validAvailRegions() []string { - return []string{"us-east-1", "us-central-1"} -} - -func validAvailZonesWithEdge() []string { - return []string{"a", "b", "c", "edge-a", "edge-b", "edge-c"} -} - -func validAvailZonesOnlyEdge() []string { - return []string{"edge-a", "edge-b", "edge-c"} -} - -func validPrivateSubnets() Subnets { - return Subnets{ - "valid-private-subnet-a": { - Zone: &Zone{Name: "a"}, - CIDR: "10.0.1.0/24", - }, - "valid-private-subnet-b": { - Zone: &Zone{Name: "b"}, - CIDR: "10.0.2.0/24", - }, - "valid-private-subnet-c": { - Zone: &Zone{Name: "c"}, - CIDR: "10.0.3.0/24", - }, - } -} - -func validPublicSubnets() Subnets { - return Subnets{ - "valid-public-subnet-a": { - Zone: &Zone{Name: "a"}, - CIDR: "10.0.4.0/24", - }, - "valid-public-subnet-b": { - Zone: &Zone{Name: "b"}, - CIDR: "10.0.5.0/24", - }, - "valid-public-subnet-c": { - Zone: &Zone{Name: "c"}, - CIDR: "10.0.6.0/24", - }, - } -} - -func validEdgeSubnets() Subnets { - return Subnets{ - "valid-public-subnet-edge-a": { - Zone: &Zone{Name: "edge-a"}, - CIDR: "10.0.7.0/24", - }, - "valid-public-subnet-edge-b": { - Zone: &Zone{Name: "edge-b"}, - CIDR: "10.0.8.0/24", - }, - "valid-public-subnet-edge-c": { - Zone: &Zone{Name: "edge-c"}, - CIDR: "10.0.9.0/24", - }, - } -} - -func validServiceEndpoints() []aws.ServiceEndpoint { - return []aws.ServiceEndpoint{{ - Name: "ec2", - URL: "e2e.local", - }, { - Name: "s3", - URL: "e2e.local", - }, { - Name: "iam", - URL: "e2e.local", - }, { - Name: "elasticloadbalancing", - URL: "e2e.local", - }, { - Name: "tagging", - URL: "e2e.local", - }, { - Name: "route53", - URL: "e2e.local", - }, { - Name: "sts", - URL: "e2e.local", - }} -} - -func invalidServiceEndpoint() []aws.ServiceEndpoint { - return []aws.ServiceEndpoint{{ - Name: "testing", - URL: "testing", - }, { - Name: "test", - URL: "http://testing.non", - }} -} - -func validInstanceTypes() map[string]InstanceType { - return map[string]InstanceType{ - "t2.small": { - DefaultVCpus: 1, - MemInMiB: 2048, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m5.large": { - DefaultVCpus: 2, - MemInMiB: 8192, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m5.xlarge": { - DefaultVCpus: 4, - MemInMiB: 16384, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m6g.xlarge": { - DefaultVCpus: 4, - MemInMiB: 16384, - Arches: []string{ec2.ArchitectureTypeArm64}, - }, - } -} - -func createBaseDomainHostedZone() route53.HostedZone { - return route53.HostedZone{ - CallerReference: &validCallerRef, - Id: &validDSId, - Name: &validDomainName, - } -} - -func createValidHostedZone() route53.GetHostedZoneOutput { - ptrValidNameServers := []*string{} - for i := range validNameServers { - ptrValidNameServers = append(ptrValidNameServers, &validNameServers[i]) - } - - validDelegationSet := route53.DelegationSet{CallerReference: &validCallerRef, Id: &validDSId, NameServers: ptrValidNameServers} - validHostedZone := route53.HostedZone{CallerReference: &validCallerRef, Id: &validDSId, Name: &validHostedZoneName} - validVPCs := []*route53.VPC{{VPCId: &validHostedZoneName, VPCRegion: &validRegion}} - - return route53.GetHostedZoneOutput{ - DelegationSet: &validDelegationSet, - HostedZone: &validHostedZone, - VPCs: validVPCs, - } -} - func TestValidate(t *testing.T) { tests := []struct { - name string - installConfig *types.InstallConfig - availZones []string - availRegions []string - edgeZones []string - privateSubnets Subnets - publicSubnets Subnets - edgeSubnets Subnets - instanceTypes map[string]InstanceType - proxy string - publicOnly string - expectErr string - }{{ - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = nil - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: validInstallConfig(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: validInstallConfigEdgeSubnets(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid instance types", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{ - Region: "us-east-1", - DefaultMachinePlatform: &aws.MachinePool{ - InstanceType: "m5.xlarge", - }, - } - c.ControlPlane.Platform.AWS.InstanceType = "m5.xlarge" - c.Compute[0].Platform.AWS.InstanceType = "m5.large" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - }, { - name: "invalid control plane instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.ControlPlane.Platform.AWS.InstanceType = "t2.small" - c.Compute[0].Platform.AWS.InstanceType = "m5.large" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Q[controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 4 vCPUs, controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 16384 MiB Memory]\E$`, - }, { - name: "invalid compute instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.ControlPlane.Platform.AWS.InstanceType = "m5.xlarge" - c.Compute[0].Platform.AWS.InstanceType = "t2.small" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Q[compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 2 vCPUs, compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 8192 MiB Memory]\E$`, - }, { - name: "undefined compute instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.Compute[0].Platform.AWS.InstanceType = "m5.dummy" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Qcompute[0].platform.aws.type: Invalid value: "m5.dummy": instance type m5.dummy not found\E$`, - }, { - name: "mismatched instance architecture", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{ - Region: "us-east-1", - DefaultMachinePlatform: &aws.MachinePool{InstanceType: "m5.xlarge"}, - } - c.ControlPlane.Architecture = types.ArchitectureARM64 - c.Compute[0].Platform.AWS.InstanceType = "m6g.xlarge" - c.Compute[0].Architecture = types.ArchitectureAMD64 - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\[controlPlane.platform.aws.type: Invalid value: "m5.xlarge": instance type supported architectures \[amd64\] do not match specified architecture arm64, compute\[0\].platform.aws.type: Invalid value: "m6g.xlarge": instance type supported architectures \[arm64\] do not match specified architecture amd64\]$`, - }, { - name: "mismatched compute pools architectures", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[0].Architecture = types.ArchitectureAMD64 - c.Compute[1].Architecture = types.ArchitectureARM64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, - }, { - name: "valid compute pools architectures", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[0].Architecture = types.ArchitectureAMD64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - }, { - name: "mismatched compute pools architectures 2", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[1].Architecture = types.ArchitectureARM64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, - }, { - name: "invalid no private subnets", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-public-subnet-a"}, - {ID: "valid-public-subnet-b"}, - {ID: "valid-public-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - publicSubnets: validPublicSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid no public subnets", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid cidr does not belong to machine CIDR", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "invalid-cidr-subnet"}) - return c - }(), - availZones: func() []string { - zones := validAvailZones() - return append(zones, "zone-for-invalid-cidr-subnet") - }(), - privateSubnets: validPrivateSubnets(), - availRegions: validAvailRegions(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["invalid-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.126.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks$`, - }, { - name: "invalid cidr does not belong to machine CIDR", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "invalid-private-cidr-subnet"}, aws.Subnet{ID: "invalid-public-cidr-subnet"}) - return c - }(), - availZones: func() []string { - zones := validAvailZones() - return append(zones, "zone-for-invalid-cidr-subnet") - }(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["invalid-private-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.126.0/24", - } - return s - }(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["invalid-public-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.127.0/24", - } - return s - }(), - expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-private-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks, platform\.aws\.vpc\.subnets\[7\]: Invalid value: \"invalid-public-cidr-subnet\": subnet's CIDR range start 192\.168\.127\.0 is outside of the specified machine networks\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid missing public subnet in a zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "no-matching-public-private-zone"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["no-matching-public-private-zone"] = Subnet{ - Zone: &Zone{Name: "f"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - publicSubnets: validPublicSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"no-matching-public-private-zone\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[f\]$`, - }, { - name: "invalid multiple private in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-private-zone-c-2"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["valid-private-zone-c-2"] = Subnet{ - Zone: &Zone{Name: "c"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - publicSubnets: validPublicSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-private-zone-c-2\": private subnet valid-private-subnet-c is also in zone c$`, - }, { - name: "invalid multiple public in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-public-zone-c-2"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: validPrivateSubnets(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["valid-public-zone-c-2"] = Subnet{ - Zone: &Zone{Name: "c"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-public-zone-c-2\": public subnet valid-public-subnet-c is also in zone c$`, - }, { - name: "invalid multiple public edge in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-public-zone-edge-c-2"}) - return c - }(), - availZones: validAvailZonesWithEdge(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - edgeSubnets: func() Subnets { - s := validEdgeSubnets() - s["valid-public-zone-edge-c-2"] = Subnet{ - Zone: &Zone{Name: "edge-c", Type: aws.LocalZoneType}, - CIDR: "10.0.9.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[9\]: Invalid value: \"valid-public-zone-edge-c-2\": edge subnet valid-public-subnet-edge-c is also in zone edge-c$`, - }, { - name: "invalid edge pool missing valid subnets", - installConfig: validInstallConfigEdgeSubnets(), - availZones: validAvailZonesWithEdge(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - edgeSubnets: Subnets{}, - expectErr: `^compute\[1\]\.platform\.aws: Required value: the provided subnets must include valid subnets for the specified edge zones$`, - }, { - name: "invalid edge pool missing zones", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{}, - }, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, - }, { - name: "invalid edge pool empty zones", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{}, + name string + installConfig *types.InstallConfig + availRegions []string + availZones []string + edgeZones []string + subnets SubnetGroups + subnetsInVPC *SubnetGroups + instanceTypes map[string]InstanceType + proxy string + publicOnly bool + expectErr string + }{ + { + name: "valid instance types", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "m5.large")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + }, + { + name: "invalid control plane instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "t2.small", "m5.large")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Q[controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 4 vCPUs, controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 16384 MiB Memory]\E$`, + }, + { + name: "invalid compute instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "t2.small")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Q[compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 2 vCPUs, compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 8192 MiB Memory]\E$`, + }, + { + name: "invalid undefined compute instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "m5.dummy")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Qcompute[0].platform.aws.type: Invalid value: "m5.dummy": instance type m5.dummy not found\E$`, + }, + { + name: "valid compute pools architectures", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64, types.ArchitectureAMD64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "invalid mismatched instance architecture", + installConfig: icBuild.build( + icBuild.withInstanceType("m5.xlarge", "", "m6g.xlarge"), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\[controlPlane.platform.aws.type: Invalid value: "m5.xlarge": instance type supported architectures \[amd64\] do not match specified architecture arm64, compute\[0\].platform.aws.type: Invalid value: "m6g.xlarge": instance type supported architectures \[arm64\] do not match specified architecture amd64\]$`, + }, + { + name: "invalid mismatched compute pools architectures", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64, types.ArchitectureARM64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, + }, + { + name: "invalid edge pool, missing zones", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, }, - }, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, - }, { - name: "invalid edge pool missing platform definition", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{}, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^\[compute\[0\]\.platform\.aws: Required value: edge compute pools are only supported on the AWS platform, compute\[0\].platform.aws: Required value: zone is required when using edge machine pools\]$`, - }, { - name: "invalid edge pool missing subnets on availability zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - edgeSubnets := validEdgeSubnets() - for subnet := range edgeSubnets { - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: aws.AWSSubnetID(subnet)}) - } - sort.Slice(c.Platform.AWS.VPC.Subnets, func(i, j int) bool { - subnets := c.Platform.AWS.VPC.Subnets - return subnets[i].ID < subnets[j].ID - }) - return c - }(), - availZones: validAvailZonesOnlyEdge(), - privateSubnets: Subnets{}, - publicSubnets: Subnets{}, - edgeSubnets: validEdgeSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid no subnet for control plane zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.ControlPlane.Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, - }, { - name: "invalid no subnet for control plane zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.ControlPlane.Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d", "e") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\", \"e\"}: No subnets provided for zones \[d e\]$`, - }, { - name: "invalid no subnet for compute[0] zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Compute[0].Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, - }, { - name: "invalid no subnet for compute zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Compute[0].Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - c.Compute = append(c.Compute, types.MachinePool{ - Architecture: types.ArchitectureAMD64, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "e"}, + }}, true), + icBuild.withControlPlaneMachinePool(types.MachinePool{}), + ), + availRegions: validAvailRegions(), + expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, + }, + { + name: "invalid edge pool, empty zones", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{}, + }, }, - }, - }) - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^\[compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\], compute\[1\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"e\"}: No subnets provided for zones \[e\]\]$`, - }, { - name: "custom region invalid service endpoints none provided", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "custom region invalid service endpoints some provided", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints()[:3] - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "custom region valid service endpoints", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI omitted for new region in standard partition", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-newregion-1" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: "platform.aws.amiID: Required value: AMI must be provided", - }, { - name: "accept platform-level AMI", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "accept AMI from default machine platform", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.Platform.AWS.DefaultMachinePlatform = &aws.MachinePool{AMIID: "custom-ami"} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "accept AMIs specified for each machine pool", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.ControlPlane.Platform.AWS.AMIID = "custom-ami" - c.Compute[0].Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI omitted for compute with no replicas", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.ControlPlane.Platform.AWS.AMIID = "custom-ami" - c.Compute[0].Replicas = ptr.To[int64](0) - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI not provided for unknown region", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^platform\.aws\.amiID: Required value: AMI must be provided$`, - }, { - name: "invalid endpoint URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.ServiceEndpoints = invalidServiceEndpoint() - c.Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "testing": Head "testing": unsupported protocol scheme "", platform.aws.serviceEndpoints[1].url: Invalid value: "http://testing.non": Head "http://testing.non": dial tcp: lookup testing.non\E.*: no such host\]$`, - }, { - name: "invalid proxy URL but valid URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.AMIID = "custom-ami" - c.Platform.AWS.ServiceEndpoints = []aws.ServiceEndpoint{{Name: "test", URL: "http://testing.com"}} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - proxy: "proxy", - }, { - name: "invalid proxy URL and invalid URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.AMIID = "custom-ami" - c.Platform.AWS.ServiceEndpoints = []aws.ServiceEndpoint{{Name: "test", URL: "http://test"}} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - proxy: "http://proxy.com", - expectErr: `^\Qplatform.aws.serviceEndpoints[0].url: Invalid value: "http://test": Head "http://test": dial tcp: lookup test\E.*: no such host$`, - }, { - name: "invalid public ipv4 pool private installation", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - c.Platform.AWS.PublicIpv4Pool = "ipv4pool-ec2-123" - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - expectErr: `^platform.aws.publicIpv4PoolId: Invalid value: "ipv4pool-ec2-123": publish strategy Internal can't be used with custom Public IPv4 Pools$`, - }, { - name: "invalid publish method for public-only subnets install", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - return c - }(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - publicOnly: "true", - expectErr: `^publish: Invalid value: \"Internal\": cluster cannot be private with public subnets$`, - }, { - name: "no public subnets specified for public-only subnets cluster", - installConfig: validInstallConfig(), - privateSubnets: validPrivateSubnets(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - expectErr: `^\[platform\.aws\.vpc\.subnets: Required value: public subnets are required for a public-only subnets cluster, platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]\]$`, - }, { - name: "no subnets specified for public-only subnets cluster", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - }, { - name: "valid public-only subnets install config", - installConfig: validInstallConfig(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - }} + }}, true), + ), + availRegions: validAvailRegions(), + expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, + }, + { + name: "invalid edge pool missing platform definition", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{}, + }}, true), + icBuild.withControlPlaneMachinePool(types.MachinePool{}), + ), + availRegions: validAvailRegions(), + expectErr: `^\[compute\[0\]\.platform\.aws: Required value: edge compute pools are only supported on the AWS platform, compute\[0\].platform.aws: Required value: zone is required when using edge machine pools\]$`, + }, + { + name: "valid service endpoints, custom region and no endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + ), + }, + { + name: "valid service endpoints, custom region and some endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + icBuild.withServiceEndpoints(validServiceEndpoints()[:3], true), + ), + }, + { + name: "valid service endpoints, custom region and all endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + icBuild.withServiceEndpoints(validServiceEndpoints(), true), + ), + }, + { + name: "invalid service endpoint URLs", + installConfig: icBuild.build( + icBuild.withServiceEndpoints(invalidServiceEndpoint(), true), + ), + availRegions: validAvailRegions(), + expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "bad-aws-endpoint": failed to connect to service endpoint url: Head "bad-aws-endpoint": dial tcp: lookup bad-aws-endpoint: no such host, platform.aws.serviceEndpoints[1].url: Invalid value: "http://bad-aws-endpoint.non": failed to connect to service endpoint url: Head "http://bad-aws-endpoint.non": dial tcp: lookup bad-aws-endpoint.non: no such host]\E$`, + }, + { + name: "valid AMI, from platform level", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withPlatformAMIID("custom-ami"), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMI, from default platform machine", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withDefaultPlatformMachine(aws.MachinePool{AMIID: "custom-ami"}), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMIs, from machine pools", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withControlPlanePlatformAMI("custom-ami"), + icBuild.withComputePlatformAMI("custom-ami", 0), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMI, omitted for compute with no replicas", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withControlPlanePlatformAMI("custom-ami"), + icBuild.withComputeReplicas(0, 0), + ), + availRegions: validAvailRegions(), + }, + { + name: "invalid AMI not provided for unknown region", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + ), + availRegions: validAvailRegions(), + expectErr: `^platform\.aws\.amiID: Required value: AMI must be provided$`, + }, + { + name: "invalid proxy URL but valid service endpoint URL", + installConfig: icBuild.build( + icBuild.withPlatformAMIID("custom-ami"), + icBuild.withServiceEndpoints(validServiceEndpoints(), true), + ), + availRegions: validAvailRegions(), + proxy: "proxy", + }, + { + name: "invalid proxy URL and invalid service endpoint URL", + installConfig: icBuild.build( + icBuild.withPlatformAMIID("custom-ami"), + icBuild.withServiceEndpoints(invalidServiceEndpoint(), true), + ), + availRegions: validAvailRegions(), + proxy: "http://proxy.com", + expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "bad-aws-endpoint": failed to connect to service endpoint url: Head "bad-aws-endpoint": dial tcp: lookup bad-aws-endpoint: no such host, platform.aws.serviceEndpoints[1].url: Invalid value: "http://bad-aws-endpoint.non": failed to connect to service endpoint url: Head "http://bad-aws-endpoint.non": dial tcp: lookup bad-aws-endpoint.non: no such host]\E$`, + }, + { + name: "invalid public ipv4 pool in private installation", + installConfig: icBuild.build( + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withPublicIPv4Pool("ipv4pool-ec2-123"), + icBuild.withVPCSubnets([]aws.Subnet{}, true), + ), + availRegions: validAvailRegions(), + expectErr: `^platform.aws.publicIpv4PoolId: Invalid value: "ipv4pool-ec2-123": publish strategy Internal can't be used with custom Public IPv4 Pools$`, + }, + { + name: "valid no byo subnets, unspecified subnet list", + installConfig: icBuild.build(), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + }, + { + name: "valid no byo subnets, empty subnet list", + installConfig: icBuild.build(icBuild.withVPCSubnets([]aws.Subnet{}, true)), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + }, + { + name: "valid no byo subnets, unspecified subnet list for public-only subnets cluster", + installConfig: icBuild.build(), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + publicOnly: true, + }, + { + name: "valid byo subnets", + installConfig: icBuild.build(icBuild.withBaseBYO()), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets, include edge subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets, private subnets only with publish internal", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets, no private subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnetIDs(validSubnets("public").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-public-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + }, + { + name: "invalid byo subnets, no public subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-private-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]$`, + }, + { + name: "invalid byo subnets, invalid cidr does not belong to machine CIDR", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"invalid-private-cidr-subnet", "invalid-public-cidr-subnet"}, false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), "zone-for-invalid-cidr-subnet"), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"invalid-private-cidr-subnet": Subnet{ + ID: "invalid-private-cidr-subnet", + Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, + CIDR: "192.168.126.0/24", + }}), + Public: mergeSubnets(validSubnets("public"), Subnets{"invalid-public-cidr-subnet": Subnet{ + ID: "invalid-public-cidr-subnet", + Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, + CIDR: "192.168.127.0/24", + }}), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-private-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks, platform\.aws\.vpc\.subnets\[7\]: Invalid value: \"invalid-public-cidr-subnet\": subnet's CIDR range start 192\.168\.127\.0 is outside of the specified machine networks\]$`, + }, + { + name: "invalid byo subnets, missing public subnet in a zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"no-matching-public-private-zone"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"no-matching-public-private-zone": Subnet{ + ID: "no-matching-public-private-zone", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.7.0/24", + }}), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-private-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"no-matching-public-private-zone\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[f\]$`, + }, + { + name: "invalid byo subnets, multiple private in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"valid-private-zone-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"valid-private-zone-c-2": Subnet{ + ID: "valid-private-zone-c-2", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.7.0/24", + }}), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-private-zone-c-2\": private subnet subnet-valid-private-c is also in zone c$`, + }, + { + name: "invalid byo subnets, multiple public in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"valid-public-zone-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{"valid-public-zone-c-2": Subnet{ + ID: "valid-public-zone-c-2", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.7.0/24", + }}), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-public-zone-c-2\": public subnet subnet-valid-public-c is also in zone c$`, + }, + { + name: "invalid byo subnets, multiple public edge in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withVPCEdgeSubnetIDs([]string{"valid-public-zone-edge-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), validEdgeAvailZones()...), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: mergeSubnets(validSubnets("edge"), Subnets{ + "valid-public-zone-edge-c-2": Subnet{ + ID: "valid-public-zone-edge-c-2", + Zone: &Zone{Name: "edge-c", Type: aws.LocalZoneType}, + CIDR: "10.0.9.0/24", + }, + }), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[9\]: Invalid value: \"valid-public-zone-edge-c-2\": edge subnet subnet-valid-public-edge-c is also in zone edge-c$`, + }, + { + name: "invalid byo subnets, edge pool missing valid subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), validEdgeAvailZones()...), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^compute\[1\]\.platform\.aws: Required value: the provided subnets must include valid subnets for the specified edge zones$`, + }, + { + name: "invalid byo subnets, edge pool missing subnets on availability zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validEdgeAvailZones(), + subnets: SubnetGroups{ + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-public-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + }, + { + name: "invalid byo subnets, no subnet for control plane zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withControlPlanePlatformZones([]string{"d", "e"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\", \"e\"}: No subnets provided for zones \[d e\]$`, + }, + { + name: "invalid byo subnets, no subnet for compute[0] zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withComputePlatformZones([]string{"d"}, false, 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, + }, + { + name: "invalid byo subnets, no subnet for compute zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withComputePlatformZones([]string{"d"}, false, 0), + icBuild.withComputeMachinePool([]types.MachinePool{{ + Architecture: types.ArchitectureAMD64, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "e"}, + }, + }, + }}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\], compute\[1\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"e\"}: No subnets provided for zones \[e\]\]$`, + }, + { + name: "valid byo subnets, private and public subnets provided for public-only subnets cluster", + installConfig: icBuild.build(icBuild.withBaseBYO()), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("public"), validSubnets("private")), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "valid byo subnets, public subnets provided for public-only subnets cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("public").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("public"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "invalid byo subnets, no public subnets specified for public-only subnets cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + publicOnly: true, + expectErr: `^\Q[platform.aws.vpc.subnets: Required value: public subnets are required for a public-only subnets cluster, platform.aws.vpc.subnets: Invalid value: []aws.Subnet{aws.Subnet{ID:"subnet-valid-private-a", Roles:[]aws.SubnetRole(nil)}, aws.Subnet{ID:"subnet-valid-private-b", Roles:[]aws.SubnetRole(nil)}, aws.Subnet{ID:"subnet-valid-private-c", Roles:[]aws.SubnetRole(nil)}}: No public subnet provided for zones [a b c]]\E$`, + }, + { + name: "invalid byo subnets, internal publish method for public-only subnets install", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + expectErr: `^publish: Invalid value: \"Internal\": cluster cannot be private with public subnets$`, + }, + { + name: "valid byo subnets, no roles and vpc has no untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherTaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets, no roles but vpc has untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherUntaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Forbidden: additional subnets \[subnet-valid-private-a1 subnet-valid-private-b1\] without tag prefix kubernetes\.io/cluster/ are found in vpc vpc-valid-id of provided subnets\. Please add a tag kubernetes\.io/cluster/unmanaged to those subnets to exclude them from cluster installation or explicitly assign roles in the install-config to provided subnets$`, + }, + { + name: "valid byo subnets with roles", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, include edge subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCEdgeSubnets(byoEdgeSubnetsWithRoles(), false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, ignore other untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherUntaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, vpc has other tagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherTaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets with roles, public subnet assigned ClusterNode", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.ClusterNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-public-a1": public subnet subnet-valid-public-a is also in zone a, platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-public-a1": subnet subnet-valid-public-a1 has role ClusterNode, but is public, expected to be private]\E$`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned Bootstrap in external cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-private-a1": private subnet subnet-valid-private-a is also in zone a, platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-private-a1": subnet subnet-valid-private-a1 has role BootstrapNode, but is private, expected to be public]\E$`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned ControlPlaneExternalLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneExternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": private subnet subnet-valid-private-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": subnet subnet-valid-private-a1 has role ControlPlaneExternalLB, but is private, expected to be public\]$`, + }, + { + name: "invalid byo subnets with roles, public subnet assigned ControlPlaneInternalLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneInternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": public subnet subnet-valid-public-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnet subnet-valid-public-a1 has role ControlPlaneInternalLB, but is public, expected to be private\]$`, + }, + { + name: "valid byo subnets with roles, public subnet assigned ControlPlaneInternalLB, ClusterNode and public-only cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoPublicOnlySubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Public: validSubnets("public"), + Private: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "invalid byo subnets with roles, public subnet assigned IngressControllerLB when publish is internal", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[3\]: Invalid value: \"subnet-valid-public-a\": subnet subnet-valid-public-a has role IngressControllerLB and is public, which is not allowed when publish is set to Internal`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned IngressControllerLB when publish is external", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.IngressControllerLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": subnet subnet-valid-private-a1 has role IngressControllerLB and is private, which is not allowed when publish is set to External`, + }, + { + name: "invalid byo subnets with roles, subnets assigned IngressControllerLB in the same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.IngressControllerLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": public subnet subnet-valid-public-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnets subnet-valid-public-a and subnet-valid-public-a1 have role IngressControllerLB and are both in zone a`, + }, + { + name: "invalid byo subnets with roles, AZs of IngressControllerLB and ControlPlaneLB not match AZs of ClusterNode", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-f", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneExternalLBSubnetRole}, {Type: aws.IngressControllerLBSubnetRole}}, + }, + { + ID: "subnet-valid-private-f", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneInternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-f": { + ID: "subnet-valid-private-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-f": { + ID: "subnet-valid-public-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for ControlPlaneInternalLB load balancers, but are not used by any nodes, platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for IngressControllerLB load balancers, but are not used by any nodes, platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for ControlPlaneExternalLB load balancers, but are not used by any nodes]\E$`, + }, + { + name: "invalid byo subnets with roles, AZs of ClusterNode not match AZs of IngressControllerLB and ControlPlaneLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-f", + Roles: []aws.SubnetRole{{Type: aws.ClusterNodeSubnetRole}}, + }, + { + ID: "subnet-valid-public-f", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-f": { + ID: "subnet-valid-private-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-f": { + ID: "subnet-valid-public-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for ControlPlaneInternalLB load balancers, nodes in those zones are unreachable, platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for IngressControllerLB load balancers, nodes in those zones are unreachable, platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for ControlPlaneExternalLB load balancers, nodes in those zones are unreachable]\E$`, + }, + { + name: "invalid byo subnets with roles, subnet assigned EdgeNode but not edge subnet", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCEdgeSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.EdgeNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnet subnet-valid-public-a1 has role EdgeNode, but is not in a Local or WaveLength Zone`, + }, + { + name: "invalid byo subnets with roles, edge subnet assigned with other roles", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-edge-a1", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: mergeSubnets(validSubnets("edge"), Subnets{ + "subnet-valid-public-edge-a1": { + ID: "subnet-valid-public-edge-a1", + Zone: &Zone{Name: "edge-a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-edge-a1\": subnet subnet-valid-public-edge-a1 must only be assigned role EdgeNode since it is in a Local or WaveLength Zone`, + }, + } + + // Register mock http(s) responses for tests. + httpmock.Activate() + t.Cleanup(httpmock.Deactivate) + + for _, endpoint := range validServiceEndpoints() { + httpmock.RegisterResponder(http.MethodHead, endpoint.URL, func(r *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(http.StatusOK, ""), nil + }) + } + + for _, endpoint := range invalidServiceEndpoint() { + httpmock.RegisterResponder(http.MethodHead, endpoint.URL, func(r *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("dial tcp: lookup %s: no such host", trimURLScheme(endpoint.URL)) + }) + } for _, test := range tests { t.Run(test.name, func(t *testing.T) { meta := &Metadata{ availabilityZones: test.availZones, availableRegions: test.availRegions, - subnets: SubnetGroups{ - Private: test.privateSubnets, - Public: test.publicSubnets, - Edge: test.edgeSubnets, - VPC: "valid-vpc", - }, - vpcSubnets: SubnetGroups{ - Private: test.privateSubnets, - Public: test.publicSubnets, - Edge: test.edgeSubnets, - VPC: "valid-vpc", - }, - vpc: "valid-vpc", - instanceTypes: test.instanceTypes, - ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets, + edgeZones: test.edgeZones, + subnets: test.subnets, + vpcSubnets: test.subnets, + vpc: validVPCID, + instanceTypes: test.instanceTypes, + ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets, + } + + if test.subnetsInVPC != nil { + meta.vpcSubnets = *test.subnetsInVPC } if test.proxy != "" { os.Setenv("HTTP_PROXY", test.proxy) } else { os.Unsetenv("HTTP_PROXY") } - if test.publicOnly != "" { - os.Setenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY", test.publicOnly) + if test.publicOnly { + os.Setenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY", "true") } else { os.Unsetenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY") } + err := Validate(context.TODO(), meta, test.installConfig) if test.expectErr == "" { assert.NoError(t, err) @@ -1040,35 +1150,35 @@ func TestIsHostedZoneDomainParentOfClusterDomain(t *testing.T) { func TestValidateForProvisioning(t *testing.T) { cases := []struct { name string - edits editFunctions + icOptions []icOption expectedErr string }{{ // This really should test for nil, as nothing happened, but no errors were provided - name: "internal publish strategy no hosted zone", - edits: editFunctions{publishInternal, clearHostedZone}, + name: "internal publish strategy no hosted zone", + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy), icBuild.withHostedZone("")}, }, { name: "external publish strategy no hosted zone invalid (empty) base domain", - edits: editFunctions{clearHostedZone, clearBaseDomain}, + icOptions: []icOption{icBuild.withHostedZone(""), icBuild.withBaseDomain("")}, expectedErr: "baseDomain: Invalid value: \"\": cannot find base domain", }, { name: "external publish strategy no hosted zone invalid base domain", - edits: editFunctions{clearHostedZone, invalidateBaseDomain}, + icOptions: []icOption{icBuild.withHostedZone(""), icBuild.withBaseDomain(invalidBaseDomain)}, expectedErr: "baseDomain: Invalid value: \"invalid-base-domain\": cannot find base domain", }, { - name: "external publish strategy no hosted zone valid base domain", - edits: editFunctions{clearHostedZone}, + name: "external publish strategy no hosted zone valid base domain", + icOptions: []icOption{icBuild.withHostedZone("")}, }, { - name: "internal publish strategy valid hosted zone", - edits: editFunctions{publishInternal}, + name: "internal publish strategy valid hosted zone", + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy)}, }, { name: "internal publish strategy invalid hosted zone", - edits: editFunctions{publishInternal, invalidateHostedZone}, + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy), icBuild.withHostedZone(invalidHostedZoneName)}, expectedErr: "aws.hostedZone: Invalid value: \"invalid-hosted-zone\": unable to retrieve hosted zone", }, { name: "external publish strategy valid hosted zone", }, { name: "external publish strategy invalid hosted zone", - edits: editFunctions{invalidateHostedZone}, + icOptions: []icOption{icBuild.withHostedZone(invalidHostedZoneName)}, expectedErr: "aws.hostedZone: Invalid value: \"invalid-hosted-zone\": unable to retrieve hosted zone", }} @@ -1077,8 +1187,8 @@ func TestValidateForProvisioning(t *testing.T) { route53Client := mock.NewMockAPI(mockCtrl) - validHostedZoneOutput := createValidHostedZone() - validDomainOutput := createBaseDomainHostedZone() + validHostedZoneOutput := createValidHostedZoneOutput() + validDomainOutput := createBaseDomainHostedZoneOutput() route53Client.EXPECT().GetBaseDomain(validDomainName).Return(&validDomainOutput, nil).AnyTimes() route53Client.EXPECT().GetBaseDomain("").Return(nil, fmt.Errorf("invalid value: \"\": cannot find base domain")).AnyTimes() @@ -1093,30 +1203,21 @@ func TestValidateForProvisioning(t *testing.T) { for _, test := range cases { t.Run(test.name, func(t *testing.T) { - editedInstallConfig := validInstallConfig() - for _, edit := range test.edits { - edit(editedInstallConfig) - } - + ic := icBuild.build(test.icOptions...) meta := &Metadata{ availabilityZones: validAvailZones(), subnets: SubnetGroups{ - Private: validPrivateSubnets(), - Public: validPublicSubnets(), - VPC: "valid-vpc", - }, - vpcSubnets: SubnetGroups{ - Private: validPrivateSubnets(), - Public: validPublicSubnets(), - VPC: "valid-vpc", + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, }, instanceTypes: validInstanceTypes(), - Region: editedInstallConfig.AWS.Region, - vpc: "valid-private-subnet-a", - ProvidedSubnets: editedInstallConfig.Platform.AWS.VPC.Subnets, + Region: ic.AWS.Region, + vpc: validVPCID, + ProvidedSubnets: ic.Platform.AWS.VPC.Subnets, } - err := ValidateForProvisioning(route53Client, editedInstallConfig, meta) + err := ValidateForProvisioning(route53Client, ic, meta) if test.expectedErr == "" { assert.NoError(t, err) } else { @@ -1155,7 +1256,7 @@ func TestGetSubDomainDNSRecords(t *testing.T) { }, } - validDomainOutput := createBaseDomainHostedZone() + validDomainOutput := createBaseDomainHostedZoneOutput() mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -1164,10 +1265,7 @@ func TestGetSubDomainDNSRecords(t *testing.T) { for _, test := range cases { t.Run(test.name, func(t *testing.T) { - - ic := validInstallConfig() - ic.BaseDomain = test.baseDomain - + ic := icBuild.build(icBuild.withBaseDomain(test.baseDomain)) if test.expectedErr != "" { if test.problematicRecords == nil { route53Client.EXPECT().GetSubDomainDNSRecords(&validDomainOutput, ic, gomock.Any()).Return(nil, fmt.Errorf("%s", test.expectedErr)).AnyTimes() @@ -1226,8 +1324,7 @@ func TestSkipRecords(t *testing.T) { } // create the dottedClusterDomain in the same manner that it will be used in GetSubDomainDNSRecords - ic := validInstallConfig() - ic.BaseDomain = validDomainName + ic := icBuild.build() dottedClusterDomain := ic.ClusterDomain() + "." for _, test := range cases { @@ -1236,3 +1333,532 @@ func TestSkipRecords(t *testing.T) { }) } } + +func validAvailRegions() []string { + return []string{"us-east-1", "us-central-1"} +} + +func validAvailZones() []string { + return []string{"a", "b", "c"} +} + +func validEdgeAvailZones() []string { + return []string{"edge-a", "edge-b", "edge-c"} +} + +func validSubnets(subnetType string) Subnets { + switch subnetType { + case "edge": + return Subnets{ + "subnet-valid-public-edge-a": { + ID: "subnet-valid-public-edge-a", + Zone: &Zone{Name: "edge-a"}, + CIDR: "10.0.7.0/24", + Public: true, + }, + "subnet-valid-public-edge-b": { + ID: "subnet-valid-public-edge-b", + Zone: &Zone{Name: "edge-b"}, + CIDR: "10.0.8.0/24", + Public: true, + }, + "subnet-valid-public-edge-c": { + ID: "subnet-valid-public-edge-c", + Zone: &Zone{Name: "edge-c"}, + CIDR: "10.0.9.0/24", + Public: true, + }, + } + case "public": + return Subnets{ + "subnet-valid-public-a": { + ID: "subnet-valid-public-a", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.4.0/24", + Public: true, + }, + "subnet-valid-public-b": { + ID: "subnet-valid-public-b", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.5.0/24", + Public: true, + }, + "subnet-valid-public-c": { + ID: "subnet-valid-public-c", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + } + case "private": + return Subnets{ + "subnet-valid-private-a": { + ID: "subnet-valid-private-a", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.1.0/24", + Public: false, + }, + "subnet-valid-private-b": { + ID: "subnet-valid-private-b", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.2.0/24", + Public: false, + }, + "subnet-valid-private-c": { + ID: "subnet-valid-private-c", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.3.0/24", + Public: false, + }, + } + } + return nil +} + +// byoSubnetsWithRoles returns a valid collection of subnets +// with assigned roles. +func byoSubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-private-a", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-private-b", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-private-c", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-a", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-b", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-c", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + {Type: aws.BootstrapNodeSubnetRole}, + }, + }, + } +} + +// byoPublicOnlySubnetsWithRoles returns a valid collection of subnets +// with assigned roles for a public-only cluster. +func byoPublicOnlySubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-public-a", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-b", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-c", + Roles: []aws.SubnetRole{ + {Type: aws.BootstrapNodeSubnetRole}, + }, + }, + } +} + +// byoEdgeSubnetsWithRoles returns a valid collection of edge subnets +// with assigned EdgeNode roles. +func byoEdgeSubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-public-edge-a", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-edge-b", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-edge-c", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + } +} + +func otherTaggedPrivateSubnets() Subnets { + return Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.4.0/24", + Tags: Tags{ + TagNameKubernetesClusterPrefix + "other-cluster": "owned", + }, + }, + "subnet-valid-private-b1": { + ID: "subnet-valid-private-b1", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.5.0/24", + Tags: Tags{ + TagNameKubernetesUnmanaged: "true", + }, + }, + } +} + +func otherUntaggedPrivateSubnets() Subnets { + return Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + "subnet-valid-private-b1": { + ID: "subnet-valid-private-b1", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.7.0/24", + }, + } +} + +func validServiceEndpoints() []aws.ServiceEndpoint { + return []aws.ServiceEndpoint{{ + Name: "ec2", + URL: "custom.ec2.us-east-1.amazonaws.com", + }, { + Name: "s3", + URL: "custom.s3.us-east-1.amazonaws.com", + }, { + Name: "iam", + URL: "custom.iam.us-east-1.amazonaws.com", + }, { + Name: "elasticloadbalancing", + URL: "custom.elasticloadbalancing.us-east-1.amazonaws.com", + }, { + Name: "tagging", + URL: "custom.tagging.us-east-1.amazonaws.com", + }, { + Name: "route53", + URL: "custom.route53.us-east-1.amazonaws.com", + }, { + Name: "sts", + URL: "custom.route53.us-east-1.amazonaws.com", + }} +} + +func invalidServiceEndpoint() []aws.ServiceEndpoint { + return []aws.ServiceEndpoint{{ + Name: "ec3", + URL: "bad-aws-endpoint", + }, { + Name: "route55", + URL: "http://bad-aws-endpoint.non", + }} +} + +func validInstanceTypes() map[string]InstanceType { + return map[string]InstanceType{ + "t2.small": { + DefaultVCpus: 1, + MemInMiB: 2048, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m5.large": { + DefaultVCpus: 2, + MemInMiB: 8192, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m5.xlarge": { + DefaultVCpus: 4, + MemInMiB: 16384, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m6g.xlarge": { + DefaultVCpus: 4, + MemInMiB: 16384, + Arches: []string{ec2.ArchitectureTypeArm64}, + }, + } +} + +func createBaseDomainHostedZoneOutput() route53.HostedZone { + return route53.HostedZone{ + CallerReference: &validCallerRef, + Id: &validDSId, + Name: &validDomainName, + } +} + +func createValidHostedZoneOutput() route53.GetHostedZoneOutput { + ptrValidNameServers := []*string{} + for i := range validNameServers { + ptrValidNameServers = append(ptrValidNameServers, &validNameServers[i]) + } + + validDelegationSet := route53.DelegationSet{CallerReference: &validCallerRef, Id: &validDSId, NameServers: ptrValidNameServers} + validHostedZone := route53.HostedZone{CallerReference: &validCallerRef, Id: &validDSId, Name: &validHostedZoneName} + validVPCs := []*route53.VPC{{VPCId: &validVPCID, VPCRegion: &validAvailRegions()[0]}} + + return route53.GetHostedZoneOutput{ + DelegationSet: &validDelegationSet, + HostedZone: &validHostedZone, + VPCs: validVPCs, + } +} + +type icOption func(*types.InstallConfig) +type icBuildForAWS struct{} + +var icBuild icBuildForAWS + +func (icBuild icBuildForAWS) build(opts ...icOption) *types.InstallConfig { + ic := &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: metaName, + }, + BaseDomain: validDomainName, + Publish: types.ExternalPublishingStrategy, + ControlPlane: &types.MachinePool{ + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }, + Compute: []types.MachinePool{{ + Name: types.MachinePoolComputeRoleName, + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }}, + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{ + {CIDR: *ipnet.MustParseCIDR(validCIDR)}, + }, + }, + Platform: types.Platform{ + AWS: &aws.Platform{ + Region: validAvailRegions()[0], + }, + }, + } + for _, opt := range opts { + if opt != nil { + opt(ic) + } + } + return ic +} + +func (icBuild icBuildForAWS) withBaseBYO() icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Platform.AWS.Zones = validAvailZones() + ic.Compute[0].Platform.AWS.Zones = validAvailZones() + + subnetIDs := append(validSubnets("public").IDs(), validSubnets("private").IDs()...) + ic.AWS.VPC.Subnets = append(ic.AWS.VPC.Subnets, subnetsFromIDs(subnetIDs)...) + ic.AWS.HostedZone = validHostedZoneName + } +} + +func (icBuild icBuildForAWS) withPublish(publish types.PublishingStrategy) icOption { + return func(ic *types.InstallConfig) { + ic.Publish = publish + } +} + +func (icBuild icBuildForAWS) withHostedZone(hostedZone string) icOption { + return func(ic *types.InstallConfig) { + ic.AWS.HostedZone = hostedZone + } +} + +func (icBuild icBuildForAWS) withBaseDomain(baseDomain string) icOption { + return func(ic *types.InstallConfig) { + ic.BaseDomain = baseDomain + } +} + +func (icBuild icBuildForAWS) withVPCSubnets(subnets []aws.Subnet, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.AWS.VPC.Subnets = subnets + } else { + ic.AWS.VPC.Subnets = append(ic.AWS.VPC.Subnets, subnets...) + } + } +} + +func (icBuild icBuildForAWS) withVPCSubnetIDs(subnetIDs []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCSubnets(subnetsFromIDs(subnetIDs), overwrite)(ic) + } +} + +func (icBuild icBuildForAWS) withVPCEdgeSubnets(subnets []aws.Subnet, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCSubnets(subnets, overwrite)(ic) + if len(subnets) > 0 { + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }}, false)(ic) + } + } +} + +func (icBuild icBuildForAWS) withVPCEdgeSubnetIDs(subnetIDs []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCEdgeSubnets(subnetsFromIDs(subnetIDs), overwrite)(ic) + } +} + +func (icBuild icBuildForAWS) withControlPlaneMachinePool(machinePool types.MachinePool) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane = &machinePool + } +} + +func (icBuild icBuildForAWS) withComputeMachinePool(machinePools []types.MachinePool, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Compute = machinePools + } else { + ic.Compute = append(ic.Compute, machinePools...) + } + } +} + +func (icBuild icBuildForAWS) withControlPlanePlatformZones(zones []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.ControlPlane.Platform.AWS.Zones = zones + } else { + ic.ControlPlane.Platform.AWS.Zones = append(ic.ControlPlane.Platform.AWS.Zones, zones...) + } + } +} + +func (icBuild icBuildForAWS) withComputePlatformZones(zones []string, overwrite bool, index int) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Compute[index].Platform.AWS.Zones = zones + } else { + ic.Compute[index].Platform.AWS.Zones = append(ic.Compute[index].Platform.AWS.Zones, zones...) + } + } +} + +func (icBuild icBuildForAWS) withControlPlanePlatformAMI(amiID string) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withComputePlatformAMI(amiID string, index int) icOption { + return func(ic *types.InstallConfig) { + ic.Compute[index].Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withComputeReplicas(replicas int64, index int) icOption { + return func(ic *types.InstallConfig) { + ic.Compute[index].Replicas = ptr.To(replicas) + } +} + +func (icBuild icBuildForAWS) withInstanceType(defaultInstanceType string, ctrPlaneInstanceType string, computeInstanceTypes ...string) icOption { + return func(ic *types.InstallConfig) { + if ic.Platform.AWS.DefaultMachinePlatform == nil { + icBuild.withDefaultPlatformMachine(aws.MachinePool{})(ic) + } + ic.Platform.AWS.DefaultMachinePlatform.InstanceType = defaultInstanceType + ic.ControlPlane.Platform.AWS.InstanceType = ctrPlaneInstanceType + for idx, instanceType := range computeInstanceTypes { + ic.Compute[idx].Platform.AWS.InstanceType = instanceType + } + } +} + +func (icBuild icBuildForAWS) withInstanceArchitecture(ctrPlaneInstanceArch types.Architecture, computeInstanceArchs ...types.Architecture) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Architecture = ctrPlaneInstanceArch + for idx, arch := range computeInstanceArchs { + ic.Compute[idx].Architecture = arch + } + } +} + +func (icBuild icBuildForAWS) withPlatformRegion(region string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.Region = region + } +} + +func (icBuild icBuildForAWS) withPlatformAMIID(amiID string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withServiceEndpoints(endpoints []aws.ServiceEndpoint, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Platform.AWS.ServiceEndpoints = endpoints + } else { + ic.Platform.AWS.ServiceEndpoints = append(ic.Platform.AWS.ServiceEndpoints, endpoints...) + } + } +} + +func (icBuild icBuildForAWS) withDefaultPlatformMachine(awsMachine aws.MachinePool) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.DefaultMachinePlatform = &awsMachine + } +} + +func (icBuild icBuildForAWS) withPublicIPv4Pool(publicIPv4Pool string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.PublicIpv4Pool = publicIPv4Pool + } +}