diff --git a/pkg/asset/installconfig/aws/metadata.go b/pkg/asset/installconfig/aws/metadata.go index 5e874de26d..ad2ed1aaea 100644 --- a/pkg/asset/installconfig/aws/metadata.go +++ b/pkg/asset/installconfig/aws/metadata.go @@ -26,15 +26,14 @@ type Metadata struct { availabilityZones []string availableRegions []string edgeZones []string - privateSubnets Subnets - publicSubnets Subnets - edgeSubnets Subnets + subnets SubnetGroups + vpcSubnets SubnetGroups vpc string instanceTypes map[string]InstanceType - Region string `json:"region,omitempty"` - Subnets []typesaws.Subnet `json:"subnets,omitempty"` - Services []typesaws.ServiceEndpoint `json:"services,omitempty"` + Region string `json:"region,omitempty"` + ProvidedSubnets []typesaws.Subnet `json:"subnets,omitempty"` + Services []typesaws.ServiceEndpoint `json:"services,omitempty"` ec2Client *ec2.Client @@ -43,7 +42,7 @@ type Metadata struct { // NewMetadata initializes a new Metadata object. func NewMetadata(region string, subnets []typesaws.Subnet, services []typesaws.ServiceEndpoint) *Metadata { - return &Metadata{Region: region, Subnets: subnets, Services: services} + return &Metadata{Region: region, ProvidedSubnets: subnets, Services: services} } // Session holds an AWS session which can be used for AWS API calls @@ -174,7 +173,7 @@ func (m *Metadata) EdgeSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Edge Subnets: %w", err) } - return m.edgeSubnets, nil + return m.subnets.Edge, nil } // SetZoneAttributes retrieves AWS Zone attributes and update required fields in zones. @@ -239,7 +238,7 @@ func (m *Metadata) PrivateSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Private Subnets: %w", err) } - return m.privateSubnets, nil + return m.subnets.Private, nil } // PublicSubnets retrieves subnet metadata indexed by subnet ID, for @@ -250,7 +249,29 @@ func (m *Metadata) PublicSubnets(ctx context.Context) (Subnets, error) { if err != nil { return nil, fmt.Errorf("error retrieving Public Subnets: %w", err) } - return m.publicSubnets, nil + return m.subnets.Public, nil +} + +// Subnets retrieves a group of subnet metadata that is indexed by subnet ID for all provided subnets. +// This includes private, public and edge subnets. +func (m *Metadata) Subnets(ctx context.Context) (SubnetGroups, error) { + err := m.populateSubnets(ctx) + if err != nil { + return m.subnets, fmt.Errorf("error retrieving all Subnets: %w", err) + } + return m.subnets, nil +} + +// VPCSubnets retrieves a group of all subnet metadata that is indexed by subnet ID in the VPC of the provided subnets. +// These include cluster subnets (i.e. provided in the installconfig) and potentially other non-cluster subnets in the VPC. +// +// This func is only used for validations. Use func Subnets to select only cluster subnets. +func (m *Metadata) VPCSubnets(ctx context.Context) (SubnetGroups, error) { + err := m.populateVPCSubnets(ctx) + if err != nil { + return m.vpcSubnets, fmt.Errorf("error retrieving Subnets in VPC: %w", err) + } + return m.vpcSubnets, nil } // VPC retrieves the VPC ID containing PublicSubnets and PrivateSubnets. @@ -269,49 +290,75 @@ func (m *Metadata) SubnetByID(ctx context.Context, subnetID string) (subnet Subn return subnet, fmt.Errorf("error retrieving subnet for ID %s: %w", subnetID, err) } - if subnet, ok := m.privateSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Private[subnetID]; ok { return subnet, nil } - if subnet, ok := m.publicSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Public[subnetID]; ok { return subnet, nil } - if subnet, ok := m.edgeSubnets[subnetID]; ok { + if subnet, ok := m.subnets.Edge[subnetID]; ok { return subnet, nil } return subnet, fmt.Errorf("no subnet found for ID %s", subnetID) } +// populateSubnets retrieves metadata for provided subnets. func (m *Metadata) populateSubnets(ctx context.Context) error { m.mutex.Lock() defer m.mutex.Unlock() - if len(m.Subnets) == 0 { + if len(m.ProvidedSubnets) == 0 { return errors.New("no subnets configured") } - if m.vpc != "" || len(m.privateSubnets) > 0 || len(m.publicSubnets) > 0 || len(m.edgeSubnets) > 0 { + subnetGroups := m.subnets + if m.vpc != "" || len(subnetGroups.Private) > 0 || len(subnetGroups.Public) > 0 || len(subnetGroups.Edge) > 0 { // Call to populate subnets has already happened return nil } - session, err := m.unlockedSession(ctx) + client, err := m.EC2Client(ctx) if err != nil { return err } - subnetIDs := make([]string, len(m.Subnets)) - for i, subnet := range m.Subnets { + subnetIDs := make([]string, len(m.ProvidedSubnets)) + for i, subnet := range m.ProvidedSubnets { subnetIDs[i] = string(subnet.ID) } - sb, err := subnets(ctx, session, m.Region, subnetIDs) + sb, err := subnets(ctx, client, subnetIDs, "") m.vpc = sb.VPC - m.privateSubnets = sb.Private - m.publicSubnets = sb.Public - m.edgeSubnets = sb.Edge + m.subnets = sb + return err +} + +// populateVPCSubnets retrieves metadata for all subnets in the VPC of provided subnets. +func (m *Metadata) populateVPCSubnets(ctx context.Context) error { + // we need to populate provided subnets to get the VPC ID. + if err := m.populateSubnets(ctx); err != nil { + return err + } + + m.mutex.Lock() + defer m.mutex.Unlock() + + vpcSubnetGroups := m.vpcSubnets + if len(vpcSubnetGroups.Private) > 0 || len(vpcSubnetGroups.Public) > 0 || len(vpcSubnetGroups.Edge) > 0 { + // Call to populate subnets has already happened + return nil + } + + client, err := m.EC2Client(ctx) + if err != nil { + return err + } + + sb, err := subnets(ctx, client, nil, m.vpc) + m.vpcSubnets = sb return err } diff --git a/pkg/asset/installconfig/aws/permissions_test.go b/pkg/asset/installconfig/aws/permissions_test.go index 2068af8c92..68f5a2fa09 100644 --- a/pkg/asset/installconfig/aws/permissions_test.go +++ b/pkg/asset/installconfig/aws/permissions_test.go @@ -5,7 +5,9 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "github.com/openshift/installer/pkg/ipnet" "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/aws" ) @@ -21,6 +23,58 @@ func basicInstallConfig() types.InstallConfig { } } +// validBYOSubnetsInstallConfig returns a valid install config for BYO subnets use case. +// Test cases can unset fields if necessary. +func validBYOSubnetsInstallConfig() *types.InstallConfig { + return &types.InstallConfig{ + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{ + {CIDR: *ipnet.MustParseCIDR(validCIDR)}, + }, + }, + BaseDomain: validDomainName, + Publish: types.ExternalPublishingStrategy, + Platform: types.Platform{ + AWS: &aws.Platform{ + Region: "us-east-1", + VPC: aws.VPC{ + Subnets: []aws.Subnet{ + {ID: "subnet-valid-private-a"}, + {ID: "subnet-valid-private-b"}, + {ID: "subnet-valid-private-c"}, + {ID: "subnet-valid-public-a"}, + {ID: "subnet-valid-public-b"}, + {ID: "subnet-valid-public-c"}, + }, + }, + HostedZone: validHostedZoneName, + }, + }, + ControlPlane: &types.MachinePool{ + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "c"}, + }, + }, + }, + Compute: []types.MachinePool{{ + Name: types.MachinePoolComputeRoleName, + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "c"}, + }, + }, + }}, + ObjectMeta: metav1.ObjectMeta{ + Name: metaName, + }, + } +} + func TestIncludesCreateInstanceRole(t *testing.T) { t.Run("Should be true when", func(t *testing.T) { t.Run("no machine types specified", func(t *testing.T) { @@ -310,28 +364,28 @@ func TestIAMRolePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create and delete shared IAM role permissions", func(t *testing.T) { t.Run("when role specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when role specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) @@ -340,7 +394,7 @@ func TestIAMRolePermissions(t *testing.T) { }) t.Run("create IAM role permissions", func(t *testing.T) { t.Run("when no existing roles and instance profiles are specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceRole) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) @@ -350,7 +404,7 @@ func TestIAMRolePermissions(t *testing.T) { t.Run("Should not include create IAM role permissions", func(t *testing.T) { t.Run("when role specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMRole: "custom-default-role", } @@ -359,7 +413,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when role specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role" ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role" requiredPerms := RequiredPermissionGroups(ic) @@ -367,7 +421,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMProfile: "custom-default-profile", } @@ -376,7 +430,7 @@ func TestIAMRolePermissions(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole) }) t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) @@ -390,14 +444,14 @@ func TestIAMProfilePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create and delete shared instance profile permissions", func(t *testing.T) { t.Run("when instance profile specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) }) t.Run("when instance profile specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) @@ -406,7 +460,7 @@ func TestIAMProfilePermissions(t *testing.T) { }) t.Run("create instance profile permissions", func(t *testing.T) { t.Run("when no existing instance profiles are specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile) assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) @@ -416,7 +470,7 @@ func TestIAMProfilePermissions(t *testing.T) { t.Run("Should not include create instance profile permissions", func(t *testing.T) { t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ IAMProfile: "custom-default-profile", } @@ -425,7 +479,7 @@ func TestIAMProfilePermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile) }) t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile" ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile" requiredPerms := RequiredPermissionGroups(ic) @@ -527,7 +581,7 @@ func TestIncludesKMSEncryptionKeys(t *testing.T) { func TestKMSKeyPermissions(t *testing.T) { t.Run("Should include KMS key permissions", func(t *testing.T) { t.Run("when KMS key specified for controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.EC2RootVolume = aws.EC2RootVolume{ KMSKeyARN: "custom-master-key", } @@ -535,7 +589,7 @@ func TestKMSKeyPermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when KMS key specified for compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.EC2RootVolume = aws.EC2RootVolume{ KMSKeyARN: "custom-worker-key", } @@ -543,7 +597,7 @@ func TestKMSKeyPermissions(t *testing.T) { assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when KMS key specified for defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ EC2RootVolume: aws.EC2RootVolume{ KMSKeyARN: "custom-default-key", @@ -556,14 +610,14 @@ func TestKMSKeyPermissions(t *testing.T) { t.Run("Should not include KMS key permissions", func(t *testing.T) { t.Run("when no machine types specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane = nil ic.Compute = nil requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys) }) t.Run("when no KMS keys specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys) @@ -575,14 +629,14 @@ func TestVPCPermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create network permissions when VPC not specified", func(t *testing.T) { t.Run("for standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateNetworking) }) t.Run("for secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" @@ -591,32 +645,32 @@ func TestVPCPermissions(t *testing.T) { }) }) t.Run("delete network permissions when VPC not specified for standard region", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteNetworking) }) t.Run("delete shared network permissions when VPC specified for standard region", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteSharedNetworking) }) }) t.Run("Should not include", func(t *testing.T) { t.Run("create network permissions when VPC specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCreateNetworking) }) t.Run("delete network permissions", func(t *testing.T) { t.Run("when VPC specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteNetworking) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteNetworking) @@ -624,14 +678,14 @@ func TestVPCPermissions(t *testing.T) { }) t.Run("delete shared network permissions", func(t *testing.T) { t.Run("when VPC not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = nil ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking) @@ -643,13 +697,13 @@ func TestVPCPermissions(t *testing.T) { func TestPrivateZonePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("create hosted zone permissions when PHZ not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateHostedZone) }) t.Run("delete hosted zone permissions when PHZ not specified on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZone = "" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteHostedZone) @@ -657,18 +711,18 @@ func TestPrivateZonePermissions(t *testing.T) { }) t.Run("Should not include", func(t *testing.T) { t.Run("create hosted zone permissions when PHZ specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCreateHostedZone) }) t.Run("delete hosted zone permissions", func(t *testing.T) { t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone) }) t.Run("when PHZ specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone) }) @@ -678,13 +732,13 @@ func TestPrivateZonePermissions(t *testing.T) { func TestPublicIPv4PoolPermissions(t *testing.T) { t.Run("Should include IPv4Pool permissions when IPv4 pool specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.PublicIpv4Pool = "custom-ipv4-pool" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionPublicIpv4Pool) }) t.Run("Should not include IPv4Pool permissions when IPv4 pool not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionPublicIpv4Pool) }) @@ -694,25 +748,25 @@ func TestBasePermissions(t *testing.T) { t.Run("Should include", func(t *testing.T) { t.Run("base create permissions", func(t *testing.T) { t.Run("on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateBase) }) t.Run("on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionCreateBase) }) }) t.Run("base delete permissions on standard regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteBase) }) }) t.Run("Should not include base delete permissions on secret regions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.Region = "us-iso-east-1" requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteBase) @@ -721,12 +775,12 @@ func TestBasePermissions(t *testing.T) { func TestDeleteIgnitionPermissions(t *testing.T) { t.Run("Should include delete ignition permissions", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionDeleteIgnitionObjects) }) t.Run("Should not include delete ignition permission when specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.BestEffortDeleteIgnition = true requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDeleteIgnitionObjects) @@ -737,7 +791,7 @@ func TestIncludesInstanceType(t *testing.T) { const instanceType = "m7a.2xlarge" t.Run("Should be true when instance type specified for", func(t *testing.T) { t.Run("defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.DefaultMachinePlatform = &aws.MachinePool{ InstanceType: instanceType, } @@ -745,20 +799,20 @@ func TestIncludesInstanceType(t *testing.T) { assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) t.Run("controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.InstanceType = instanceType requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) t.Run("compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.InstanceType = instanceType requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionValidateInstanceType) }) }) t.Run("Should be false when instance type is not set", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() assert.NotContains(t, RequiredPermissionGroups(ic), PermissionValidateInstanceType) }) } @@ -766,7 +820,7 @@ func TestIncludesInstanceType(t *testing.T) { func TestIncludesZones(t *testing.T) { t.Run("Should be true when", func(t *testing.T) { t.Run("zones specified in defaultMachinePlatform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} @@ -777,21 +831,21 @@ func TestIncludesZones(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("zones specified in controlPlane", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute[0].Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("zones specified in compute", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.AWS.VPC.Subnets = []aws.Subnet{} requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionDefaultZones) }) t.Run("subnets specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} requiredPerms := RequiredPermissionGroups(ic) @@ -799,7 +853,7 @@ func TestIncludesZones(t *testing.T) { }) }) t.Run("Should be false when neither zones nor subnets specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.VPC.Subnets = []aws.Subnet{} ic.ControlPlane.Platform.AWS.Zones = []string{} ic.Compute[0].Platform.AWS.Zones = []string{} @@ -810,13 +864,13 @@ func TestIncludesZones(t *testing.T) { func TestIncludesAssumeRole(t *testing.T) { t.Run("Should be true when IAM role specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.AWS.HostedZoneRole = "custom-role" requiredPerms := RequiredPermissionGroups(ic) assert.Contains(t, requiredPerms, PermissionAssumeRole) }) t.Run("Should be false when IAM role not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionAssumeRole) }) @@ -824,7 +878,7 @@ func TestIncludesAssumeRole(t *testing.T) { func TestIncludesWavelengthZones(t *testing.T) { t.Run("Should be true when edge compute specified with WL zones", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -838,7 +892,7 @@ func TestIncludesWavelengthZones(t *testing.T) { }) t.Run("Should be false when", func(t *testing.T) { t.Run("edge compute specified without WL zones", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -851,7 +905,7 @@ func TestIncludesWavelengthZones(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionCarrierGateway) }) t.Run("edge compute not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionCarrierGateway) }) @@ -861,7 +915,7 @@ func TestIncludesWavelengthZones(t *testing.T) { func TestIncludesEdgeDefaultInstance(t *testing.T) { t.Run("Should be true when at least one edge compute pool specified", func(t *testing.T) { t.Run("without platform", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", }) @@ -878,7 +932,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { assert.Contains(t, requiredPerms, PermissionEdgeDefaultInstance) }) t.Run("without instance type", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -902,7 +956,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { }) t.Run("Should be false when", func(t *testing.T) { t.Run("edge compute specified with instance type", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() ic.Compute = append(ic.Compute, types.MachinePool{ Name: "edge", Platform: types.MachinePoolPlatform{ @@ -916,7 +970,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) { assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance) }) t.Run("edge compute not specified", func(t *testing.T) { - ic := validInstallConfig() + ic := validBYOSubnetsInstallConfig() requiredPerms := RequiredPermissionGroups(ic) assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance) }) diff --git a/pkg/asset/installconfig/aws/subnet.go b/pkg/asset/installconfig/aws/subnet.go index a1fd38095d..601b6c4c18 100644 --- a/pkg/asset/installconfig/aws/subnet.go +++ b/pkg/asset/installconfig/aws/subnet.go @@ -3,12 +3,13 @@ package aws import ( "context" "fmt" + "maps" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/sirupsen/logrus" + "k8s.io/utils/ptr" typesaws "github.com/openshift/installer/pkg/types/aws" ) @@ -29,11 +30,23 @@ type Subnet struct { // Public is the flag to define the subnet public. Public bool + + // Tags is the map of the subnet's tags. + Tags Tags } // Subnets is the map for the Subnet metadata indexed by subnetID. type Subnets map[string]Subnet +// IDs returns the subnet IDs (i.e. map keys) in the Subnets. +func (sns Subnets) IDs() []string { + subnetIDs := make([]string, 0) + for id := range sns { + subnetIDs = append(subnetIDs, id) + } + return subnetIDs +} + // SubnetsByZone is the map for the Subnet metadata indexed by zone. type SubnetsByZone map[string]Subnet @@ -45,92 +58,82 @@ type SubnetGroups struct { VPC string } -// subnets retrieves metadata for the given subnet(s). -func subnets(ctx context.Context, session *session.Session, region string, ids []string) (subnetGroups SubnetGroups, err error) { - metas := make(Subnets, len(ids)) - zoneNames := make([]*string, len(ids)) - availabilityZones := make(map[string]*ec2.AvailabilityZone, len(ids)) - subnetGroups = SubnetGroups{ - Public: make(Subnets, len(ids)), - Private: make(Subnets, len(ids)), - Edge: make(Subnets, len(ids)), +// subnets retrieves metadata for the given subnet(s) or VPC. +func subnets(ctx context.Context, client *ec2.Client, subnetIDs []string, vpcID string) (SubnetGroups, error) { + metas := make(Subnets, len(subnetIDs)) + zoneNames := make([]string, 0) + availabilityZones := make(map[string]ec2types.AvailabilityZone, len(subnetIDs)) + subnetGroups := SubnetGroups{ + Public: make(Subnets, len(subnetIDs)), + Private: make(Subnets, len(subnetIDs)), + Edge: make(Subnets, len(subnetIDs)), } - var vpcFromSubnet string - client := ec2.New(session, aws.NewConfig().WithRegion(region)) - - idPointers := make([]*string, 0, len(ids)) - for _, id := range ids { - idPointers = append(idPointers, aws.String(id)) + subnetInput := &ec2.DescribeSubnetsInput{} + if len(vpcID) > 0 { + subnetInput.Filters = append(subnetInput.Filters, ec2types.Filter{ + Name: ptr.To("vpc-id"), + Values: []string{vpcID}, + }) + } + if len(subnetIDs) > 0 { + subnetInput.SubnetIds = subnetIDs } - var lastError error - err = client.DescribeSubnetsPagesWithContext( - ctx, - &ec2.DescribeSubnetsInput{SubnetIds: idPointers}, - func(results *ec2.DescribeSubnetsOutput, lastPage bool) bool { - for _, subnet := range results.Subnets { - if subnet.SubnetId == nil { - continue - } - if subnet.SubnetArn == nil { - lastError = fmt.Errorf("%s has no ARN", *subnet.SubnetId) - return false - } - if subnet.VpcId == nil { - lastError = fmt.Errorf("%s has no VPC", *subnet.SubnetId) - return false - } - if subnet.AvailabilityZone == nil { - lastError = fmt.Errorf("%s has not availability zone", *subnet.SubnetId) - return false - } - - if subnetGroups.VPC == "" { - subnetGroups.VPC = *subnet.VpcId - vpcFromSubnet = *subnet.SubnetId - } else if *subnet.VpcId != subnetGroups.VPC { - lastError = fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC) - return false - } - metas[aws.StringValue(subnet.SubnetId)] = Subnet{ - ID: aws.StringValue(subnet.SubnetId), - ARN: aws.StringValue(subnet.SubnetArn), - Zone: &Zone{Name: aws.StringValue(subnet.AvailabilityZone)}, - CIDR: aws.StringValue(subnet.CidrBlock), - Public: false, - } - zoneNames = append(zoneNames, subnet.AvailabilityZone) + err := describeSubnets(ctx, client, subnetInput, func(subnets []ec2types.Subnet) error { + var vpcFromSubnet string + for _, subnet := range subnets { + if subnet.SubnetId == nil { + continue } - return !lastPage - }, - ) - if err == nil { - err = lastError - } + if len(ptr.Deref(subnet.SubnetArn, "")) == 0 { + return fmt.Errorf("%s has no ARN", *subnet.SubnetId) + } + if len(ptr.Deref(subnet.VpcId, "")) == 0 { + return fmt.Errorf("%s has no VPC", *subnet.SubnetId) + } + if len(ptr.Deref(subnet.AvailabilityZone, "")) == 0 { + return fmt.Errorf("%s has no availability zone", *subnet.SubnetId) + } + if subnetGroups.VPC == "" { + subnetGroups.VPC = *subnet.VpcId + vpcFromSubnet = *subnet.SubnetId + } else if *subnet.VpcId != subnetGroups.VPC { + return fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC) + } + + // At this point, we should be safe to dereference these fields. + metas[*subnet.SubnetId] = Subnet{ + ID: *subnet.SubnetId, + ARN: *subnet.SubnetArn, + Zone: &Zone{Name: *subnet.AvailabilityZone}, + CIDR: ptr.Deref(subnet.CidrBlock, ""), + Public: false, + Tags: FromAWSTags(subnet.Tags), + } + zoneNames = append(zoneNames, *subnet.AvailabilityZone) + } + return nil + }) if err != nil { - return subnetGroups, fmt.Errorf("describing subnets: %w", err) + return subnetGroups, err } - var routeTables []*ec2.RouteTable - err = client.DescribeRouteTablesPagesWithContext( - ctx, - &ec2.DescribeRouteTablesInput{ - Filters: []*ec2.Filter{{ - Name: aws.String("vpc-id"), - Values: []*string{aws.String(subnetGroups.VPC)}, - }}, - }, - func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool { - routeTables = append(routeTables, results.RouteTables...) - return !lastPage - }, - ) + var routeTables []ec2types.RouteTable + err = describeRouteTables(ctx, client, &ec2.DescribeRouteTablesInput{ + Filters: []ec2types.Filter{{ + Name: ptr.To("vpc-id"), + Values: []string{subnetGroups.VPC}, + }}, + }, func(rTables []ec2types.RouteTable) error { + routeTables = append(routeTables, rTables...) + return nil + }) if err != nil { - return subnetGroups, fmt.Errorf("describing route tables: %w", err) + return subnetGroups, err } - azs, err := client.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames}) + azs, err := client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames}) if err != nil { return subnetGroups, fmt.Errorf("describing availability zones: %w", err) } @@ -140,6 +143,14 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ publicOnlySubnets := typesaws.IsPublicOnlySubnetsEnabled() + var ids []string + if len(vpcID) > 0 { + ids = metas.IDs() + } + if len(subnetIDs) > 0 { + ids = subnetIDs + } + for _, id := range ids { meta, ok := metas[id] if !ok { @@ -157,10 +168,10 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ return subnetGroups, fmt.Errorf("unable to read properties of zone name %s from the list %v: %w", zoneName, zoneNames, err) } zone := availabilityZones[zoneName] - meta.Zone.Type = aws.StringValue(zone.ZoneType) - meta.Zone.GroupName = aws.StringValue(zone.GroupName) + meta.Zone.Type = ptr.Deref(zone.ZoneType, "") + meta.Zone.GroupName = ptr.Deref(zone.GroupName, "") if availabilityZones[zoneName].ParentZoneName != nil { - meta.Zone.ParentZoneName = aws.StringValue(zone.ParentZoneName) + meta.Zone.ParentZoneName = ptr.Deref(zone.ParentZoneName, "") } // AWS Local Zones are grouped as Edge subnets @@ -189,12 +200,12 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [ } // https://github.com/kubernetes/kubernetes/blob/9f036cd43d35a9c41d7ac4ca82398a6d0bef957b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L3376-L3419 -func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { - var subnetTable *ec2.RouteTable +func isSubnetPublic(rt []ec2types.RouteTable, subnetID string) (bool, error) { + var subnetTable *ec2types.RouteTable for _, table := range rt { for _, assoc := range table.Associations { - if aws.StringValue(assoc.SubnetId) == subnetID { - subnetTable = table + if ptr.Equal(assoc.SubnetId, &subnetID) { + subnetTable = &table break } } @@ -205,10 +216,10 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { // associated with the VPC's main routing table. for _, table := range rt { for _, assoc := range table.Associations { - if aws.BoolValue(assoc.Main) { + if ptr.Deref(assoc.Main, false) { logrus.Debugf("Assuming implicit use of main routing table %s for %s", - aws.StringValue(table.RouteTableId), subnetID) - subnetTable = table + ptr.Deref(table.RouteTableId, ""), subnetID) + subnetTable = &table break } } @@ -226,13 +237,56 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) { // from the default in-subnet route which is called "local" // or other virtual gateway (starting with vgv) // or vpc peering connections (starting with pcx). - if strings.HasPrefix(aws.StringValue(route.GatewayId), "igw") { + if strings.HasPrefix(ptr.Deref(route.GatewayId, ""), "igw") { return true, nil } - if strings.HasPrefix(aws.StringValue(route.CarrierGatewayId), "cagw") { + if strings.HasPrefix(ptr.Deref(route.CarrierGatewayId, ""), "cagw") { return true, nil } } return false, nil } + +// describeSubnets retrieves metadata for subnets with given filters. +func describeSubnets(ctx context.Context, client *ec2.Client, input *ec2.DescribeSubnetsInput, fn func(subnets []ec2types.Subnet) error) error { + paginator := ec2.NewDescribeSubnetsPaginator(client, input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return fmt.Errorf("describing subnets: %w", err) + } + + // If the handler returns an error, we stop early to avoid extra API calls. + if err = fn(page.Subnets); err != nil { + return err + } + } + return nil +} + +// describeRouteTables retrieves metadata for route tables with given filters. +func describeRouteTables(ctx context.Context, client *ec2.Client, input *ec2.DescribeRouteTablesInput, fn func(subnets []ec2types.RouteTable) error) error { + paginator := ec2.NewDescribeRouteTablesPaginator(client, input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return fmt.Errorf("describing route tables: %w", err) + } + + // If the handler returns an error, we stop early to avoid extra API calls. + if err = fn(page.RouteTables); err != nil { + return err + } + } + return nil +} + +// mergeSubnets merged two or more Subnets into a single one for convenience. +func mergeSubnets(groups ...Subnets) Subnets { + subnets := make(Subnets) + for _, group := range groups { + maps.Copy(subnets, group) + } + return subnets +} diff --git a/pkg/asset/installconfig/aws/tags.go b/pkg/asset/installconfig/aws/tags.go new file mode 100644 index 0000000000..38f314fa84 --- /dev/null +++ b/pkg/asset/installconfig/aws/tags.go @@ -0,0 +1,44 @@ +package aws + +import ( + "strings" + + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/utils/ptr" +) + +const ( + // TagNameKubernetesClusterPrefix is the tag name prefix used by CCM + // to differentiate multiple logically independent clusters running in the same AZ. + TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/" + + // TagNameKubernetesUnmanaged is the tag name to indicate that a resource is unmanaged + // by the cluster and should be ignored by CCM. For example, kubernetes.io/cluster/unmanaged=true. + TagNameKubernetesUnmanaged = TagNameKubernetesClusterPrefix + "unmanaged" +) + +// Tags represents AWS resource tags as a map. +// This helps avoid iterating over the tag list for every lookup. +type Tags map[string]string + +// FromAWSTags converts a list of AWS tags into a map. +func FromAWSTags(awsTags []types.Tag) Tags { + tags := make(Tags, len(awsTags)) + for _, tag := range awsTags { + key, value := ptr.Deref(tag.Key, ""), ptr.Deref(tag.Value, "") + if len(key) > 0 { + tags[key] = value + } + } + return tags +} + +// HasTagKeyPrefix returns true if there is a tag with a given key prefix. +func (t Tags) HasTagKeyPrefix(prefix string) bool { + for key := range t { + if strings.HasPrefix(key, prefix) { + return true + } + } + return false +} diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go index fc7a4e51e3..cee76e9964 100644 --- a/pkg/asset/installconfig/aws/validation.go +++ b/pkg/asset/installconfig/aws/validation.go @@ -50,7 +50,7 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig) allErrs = append(allErrs, validateAMI(ctx, meta, config)...) allErrs = append(allErrs, validatePublicIpv4Pool(ctx, meta, field.NewPath("platform", "aws", "publicIpv4PoolId"), config)...) - allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config.Platform.AWS, config.Networking, config.Publish)...) + allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config)...) if awstypes.IsPublicOnlySubnetsEnabled() { logrus.Warnln("Public-only subnets install. Please be warned this is not supported") @@ -98,8 +98,9 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig) return allErrs.ToAggregate() } -func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList { +func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList { allErrs := field.ErrorList{} + platform := config.Platform.AWS allErrs = append(allErrs, validateServiceEndpoints(fldPath.Child("serviceEndpoints"), platform.Region, platform.ServiceEndpoints)...) @@ -109,7 +110,7 @@ func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, } if len(platform.VPC.Subnets) > 0 { - allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), platform.VPC.Subnets, networking, publish)...) + allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), config)...) } if platform.DefaultMachinePlatform != nil { allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("defaultMachinePlatform"), platform, platform.DefaultMachinePlatform, controlPlaneReq, "", "")...) @@ -218,67 +219,139 @@ func validatePublicIpv4Pool(ctx context.Context, meta *Metadata, fldPath *field. return nil } -func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, subnets []awstypes.Subnet, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList { - allErrs := field.ErrorList{} - privateSubnets, err := meta.PrivateSubnets(ctx) - if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) +// subnetData holds a subnet information collected from install config and AWS API for validations. +type subnetData struct { + // The subnet index in the install config. + Idx int + // The subnet assigned roles in the install config. + Roles []awstypes.SubnetRole + // The subnet metadata from AWS API. + Subnet +} + +// subnetDataGroups is a collection of subnet information +// grouped by subnet type (i.e. public, private, and edge) and indexed by subnetIDs for validations. +type subnetDataGroups struct { + Public map[string]subnetData + Private map[string]subnetData + Edge map[string]subnetData + // A convenient alias that contains all information for all subnets. + All map[string]subnetData +} + +// Converts subnetGroups (i.e. provided subnets) to subnetDataGroups to include additional information +// from the install-config such as index and roles for validations. +func (sdg *subnetDataGroups) From(ctx context.Context, meta *Metadata, providedSubnets []awstypes.Subnet) error { + if sdg.Private == nil { + sdg.Private = make(map[string]subnetData) } - privateSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := privateSubnets[string(subnet.ID)]; ok { - privateSubnetsIdx[string(subnet.ID)] = idx - } + if sdg.Public == nil { + sdg.Public = make(map[string]subnetData) } - if len(privateSubnets) == 0 { - allErrs = append(allErrs, field.Invalid(fldPath, subnets, "No private subnets found")) + if sdg.Edge == nil { + sdg.Edge = make(map[string]subnetData) + } + if sdg.All == nil { + sdg.All = make(map[string]subnetData) } - publicSubnets, err := meta.PublicSubnets(ctx) + subnets, err := meta.Subnets(ctx) if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) + return err } - if publish == types.InternalPublishingStrategy && len(publicSubnets) > 0 { - logrus.Warnf("Public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy) - } - publicSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := publicSubnets[string(subnet.ID)]; ok { - publicSubnetsIdx[string(subnet.ID)] = idx + + for idx, subnet := range providedSubnets { + var subnetDataGroup map[string]subnetData + var subnetMeta Subnet + + if awsSubnet, ok := subnets.Private[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Private + subnetMeta = awsSubnet } + + if awsSubnet, ok := subnets.Public[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Public + subnetMeta = awsSubnet + } + + if awsSubnet, ok := subnets.Edge[string(subnet.ID)]; ok { + subnetDataGroup = sdg.Edge + subnetMeta = awsSubnet + } + + if subnetDataGroup == nil { + // Should not occur but safe against panics + continue + } + + subnetData := subnetData{ + Subnet: subnetMeta, + Idx: idx, + Roles: subnet.Roles, + } + subnetDataGroup[string(subnet.ID)] = subnetData + sdg.All[string(subnet.ID)] = subnetData } - if len(publicSubnets) == 0 && awstypes.IsPublicOnlySubnetsEnabled() { + + return nil +} + +// validateSubnets ensures BYO subnets are valid. +func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList { + allErrs := field.ErrorList{} + networking := config.Networking + providedSubnets := config.AWS.VPC.Subnets + publish := config.Publish + + subnetDataGroups := subnetDataGroups{} + if err := subnetDataGroups.From(ctx, meta, providedSubnets); err != nil { + return append(allErrs, field.Invalid(fldPath, providedSubnets, err.Error())) + } + + publicOnlySubnet := awstypes.IsPublicOnlySubnetsEnabled() + + if publicOnlySubnet && len(subnetDataGroups.Public) == 0 { allErrs = append(allErrs, field.Required(fldPath, "public subnets are required for a public-only subnets cluster")) } - edgeSubnets, err := meta.EdgeSubnets(ctx) - if err != nil { - return append(allErrs, field.Invalid(fldPath, subnets, err.Error())) + if !publicOnlySubnet && len(subnetDataGroups.Private) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, "no private subnets found")) } - edgeSubnetsIdx := map[string]int{} - for idx, subnet := range subnets { - if _, ok := edgeSubnets[string(subnet.ID)]; ok { - edgeSubnetsIdx[string(subnet.ID)] = idx + + if publish == types.InternalPublishingStrategy && len(subnetDataGroups.Public) > 0 { + logrus.Warnf("public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy) + } + + subnetsWithRole := make(map[awstypes.SubnetRoleType][]subnetData) + for _, subnet := range providedSubnets { + for _, role := range subnet.Roles { + subnetsWithRole[role.Type] = append(subnetsWithRole[role.Type], subnetDataGroups.All[string(subnet.ID)]) } } - allErrs = append(allErrs, validateSubnetCIDR(fldPath, privateSubnets, privateSubnetsIdx, networking.MachineNetwork)...) - allErrs = append(allErrs, validateSubnetCIDR(fldPath, publicSubnets, publicSubnetsIdx, networking.MachineNetwork)...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, privateSubnets, privateSubnetsIdx, "private")...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, publicSubnets, publicSubnetsIdx, "public")...) - allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, edgeSubnets, edgeSubnetsIdx, "edge")...) + allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Private, networking.MachineNetwork)...) + allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Public, networking.MachineNetwork)...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Private, "private")...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Public, "public")...) + allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Edge, "edge")...) + + if len(subnetsWithRole) > 0 { + allErrs = append(allErrs, validateSubnetRoles(fldPath, subnetsWithRole, subnetDataGroups, config)...) + } else { + allErrs = append(allErrs, validateUntaggedSubnets(ctx, fldPath, meta, subnetDataGroups)...) + } privateZones := sets.New[string]() publicZones := sets.New[string]() - for _, subnet := range privateSubnets { + for _, subnet := range subnetDataGroups.Private { privateZones.Insert(subnet.Zone.Name) } - for _, subnet := range publicSubnets { + for _, subnet := range subnetDataGroups.Public { publicZones.Insert(subnet.Zone.Name) } if publish == types.ExternalPublishingStrategy && !publicZones.IsSuperset(privateZones) { errMsg := fmt.Sprintf("No public subnet provided for zones %s", sets.List(privateZones.Difference(publicZones))) - allErrs = append(allErrs, field.Invalid(fldPath, subnets, errMsg)) + allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, errMsg)) } return allErrs @@ -436,11 +509,11 @@ func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *fiel return allErrs } -func validateSubnetCIDR(fldPath *field.Path, subnets Subnets, idxMap map[string]int, networks []types.MachineNetworkEntry) field.ErrorList { +func validateSubnetCIDR(fldPath *field.Path, subnetDataGroup map[string]subnetData, networks []types.MachineNetworkEntry) field.ErrorList { allErrs := field.ErrorList{} - for id, v := range subnets { - fp := fldPath.Index(idxMap[id]) - cidr, _, err := net.ParseCIDR(v.CIDR) + for id, subnetData := range subnetDataGroup { + fp := fldPath.Index(subnetData.Idx) + cidr, _, err := net.ParseCIDR(subnetData.CIDR) if err != nil { allErrs = append(allErrs, field.Invalid(fp, id, err.Error())) continue @@ -459,27 +532,215 @@ func validateMachineNetworksContainIP(fldPath *field.Path, networks []types.Mach return field.ErrorList{field.Invalid(fldPath, subnetName, fmt.Sprintf("subnet's CIDR range start %s is outside of the specified machine networks", ip))} } -func validateDuplicateSubnetZones(fldPath *field.Path, subnets Subnets, idxMap map[string]int, typ string) field.ErrorList { - var keys []string - for id := range subnets { - keys = append(keys, id) +func validateDuplicateSubnetZones(fldPath *field.Path, subnetDataGroup map[string]subnetData, typ string) field.ErrorList { + subnetIDs := make([]string, 0) + for id := range subnetDataGroup { + subnetIDs = append(subnetIDs, id) } - sort.Strings(keys) + sort.Strings(subnetIDs) allErrs := field.ErrorList{} zones := map[string]string{} - for _, id := range keys { - subnet := subnets[id] - if conflictingSubnet, ok := zones[subnet.Zone.Name]; ok { - errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnet.Zone.Name) - allErrs = append(allErrs, field.Invalid(fldPath.Index(idxMap[id]), id, errMsg)) + for _, id := range subnetIDs { + subnetData := subnetDataGroup[id] + if conflictingSubnet, ok := zones[subnetData.Zone.Name]; ok { + errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnetData.Zone.Name) + allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), id, errMsg)) } else { - zones[subnet.Zone.Name] = id + zones[subnetData.Zone.Name] = id } } return allErrs } +// validateSubnetRoles ensures BYO subnets have valid roles assigned if roles are provided. +func validateSubnetRoles(fldPath *field.Path, subnetsWithRole map[awstypes.SubnetRoleType][]subnetData, subnetDataGroups subnetDataGroups, config *types.InstallConfig) field.ErrorList { + allErrs := field.ErrorList{} + + // BootstrapNode subnets must be assigned to public subnets + // in external cluster. + for _, bstrSubnet := range subnetsWithRole[awstypes.BootstrapNodeSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[bstrSubnet.ID]; ok { + continue + } + if config.Publish == types.ExternalPublishingStrategy && !bstrSubnet.Public { + allErrs = append(allErrs, field.Invalid(fldPath.Index(bstrSubnet.Idx), bstrSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", bstrSubnet.ID, awstypes.BootstrapNodeSubnetRole))) + } + } + + // ClusterNode subnets must be assigned to private subnets + // unless cluster is public-only. + for _, cnSubnet := range subnetsWithRole[awstypes.ClusterNodeSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[cnSubnet.ID]; ok { + continue + } + if cnSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() { + allErrs = append(allErrs, field.Invalid(fldPath.Index(cnSubnet.Idx), cnSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", cnSubnet.ID, awstypes.ClusterNodeSubnetRole))) + } + } + + // Type of ControlPlaneLB subnets must match its scope: + // - ControlPlaneInternalLB subnets must be private + // - ControlPlaneExternalLB subnets must be public. + // Private cluster must not have ControlPlaneExternalLB subnets (i.e. statically validated in pkg/types/aws/validation/platform.go). + for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneInternalLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok { + continue + } + if ctrlPSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() { + allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", ctrlPSubnet.ID, awstypes.ControlPlaneInternalLBSubnetRole))) + } + } + for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneExternalLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok { + continue + } + if !ctrlPSubnet.Public { + allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", ctrlPSubnet.ID, awstypes.ControlPlaneExternalLBSubnetRole))) + } + } + + // Type of IngressControllerLB subnets must match cluster scope: + // - In public cluster, only public IngressControllerLB subnets is allowed. + // - In private cluster, only private IngressControllerLB subnets is allowed. + for _, ingressSubnet := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] { + // We validate edge subnets in subsequent validations. + if _, ok := subnetDataGroups.Edge[ingressSubnet.ID]; ok { + continue + } + + if ingressSubnet.Public != config.PublicIngress() { + subnetType := "private" + if ingressSubnet.Public { + subnetType = "public" + } + allErrs = append(allErrs, field.Invalid(fldPath.Index(ingressSubnet.Idx), ingressSubnet.ID, + fmt.Sprintf("subnet %s has role %s and is %s, which is not allowed when publish is set to %s", ingressSubnet.ID, awstypes.IngressControllerLBSubnetRole, subnetType, config.Publish))) + } + } + + // IngressControllerLB subnets must be in different AZs as required by AWS CCM. + ingressZones := make(map[string]string) + for _, subnetData := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] { + if conflictingSubnet, ok := ingressZones[subnetData.Zone.Name]; ok { + allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), subnetData.ID, + fmt.Sprintf("subnets %s and %s have role %s and are both in zone %s", conflictingSubnet, subnetData.ID, awstypes.IngressControllerLBSubnetRole, subnetData.Zone.Name))) + } else { + ingressZones[subnetData.Zone.Name] = subnetData.ID + } + } + + // AZs of LB subnets match AZs of ClusterNode subnets. + lbRoles := []awstypes.SubnetRoleType{ + awstypes.ControlPlaneInternalLBSubnetRole, + awstypes.IngressControllerLBSubnetRole, + } + if config.Publish == types.ExternalPublishingStrategy { + lbRoles = append(lbRoles, awstypes.ControlPlaneExternalLBSubnetRole) + } + for _, role := range lbRoles { + allErrs = append(allErrs, validateLBSubnetAZMatchClusterNodeAZ(fldPath, subnetDataGroups, role, subnetsWithRole[role], subnetsWithRole[awstypes.ClusterNodeSubnetRole])...) + } + + // EdgeNode subnets must be subnets in Local or Wavelength Zones. + for _, edgeSubnet := range subnetsWithRole[awstypes.EdgeNodeSubnetRole] { + if _, ok := subnetDataGroups.Edge[edgeSubnet.ID]; !ok { + allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID, + fmt.Sprintf("subnet %s has role %s, but is not in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole))) + } + } + + // Subnets that are in Local or Wavelength Zones must only have EdgeNode role. + for _, edgeSubnet := range subnetDataGroups.Edge { + for _, role := range edgeSubnet.Roles { + if role.Type != awstypes.EdgeNodeSubnetRole { + allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID, + fmt.Sprintf("subnet %s must only be assigned role %s since it is in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole))) + break + } + } + } + + return allErrs +} + +// validateUntaggedSubnets ensures there are no additional untagged subnets in the BYO VPC. +// An untagged subnet is a subnet without tag kubernetes.io/cluster/. +// Untagged subnets may be selected by the CCM, leading to various bugs, RFEs, and support cases. See: +// - https://issues.redhat.com/browse/OCPBUGS-17432. +// - https://issues.redhat.com/browse/RFE-2816. +func validateUntaggedSubnets(ctx context.Context, fldPath *field.Path, meta *Metadata, subnetDataGroups subnetDataGroups) field.ErrorList { + allErrs := field.ErrorList{} + + vpcSubnets, err := meta.VPCSubnets(ctx) + if err != nil { + return append(allErrs, field.Invalid(fldPath, meta.ProvidedSubnets, err.Error())) + } + + untaggedSubnetIDs := make([]string, 0) + for _, subnet := range mergeSubnets(vpcSubnets.Public, vpcSubnets.Private, vpcSubnets.Edge) { + // We only check other subnets in the VPC that are not provided in the install-config. + if _, ok := subnetDataGroups.All[subnet.ID]; !ok && !subnet.Tags.HasTagKeyPrefix(TagNameKubernetesClusterPrefix) { + untaggedSubnetIDs = append(untaggedSubnetIDs, subnet.ID) + } + } + sort.Strings(untaggedSubnetIDs) + + if len(untaggedSubnetIDs) > 0 { + errMsg := fmt.Sprintf("additional subnets %v without tag prefix %s are found in vpc %s of provided subnets. %s", untaggedSubnetIDs, TagNameKubernetesClusterPrefix, vpcSubnets.VPC, + fmt.Sprintf("Please add a tag %s to those subnets to exclude them from cluster installation or explicitly assign roles in the install-config to provided subnets", TagNameKubernetesUnmanaged)) + allErrs = append(allErrs, field.Forbidden(fldPath, errMsg)) + } + + return allErrs +} + +// validateLBSubnetAZMatchClusterNodeAZ ensures AZs of LB subnets match AZs of ClusterNode subnets. +// AWS load balancers will NOT register a node located in an AZ that is not enabled for the load balancer. +func validateLBSubnetAZMatchClusterNodeAZ(fldPath *field.Path, subnetDataGroups subnetDataGroups, lbType awstypes.SubnetRoleType, lbSubnets []subnetData, clusterNodeSubnets []subnetData) field.ErrorList { + allErrs := field.ErrorList{} + + lbZoneSet := sets.New[string]() + for _, subnet := range lbSubnets { + // We validate edge subnets in another place. + if _, ok := subnetDataGroups.Edge[subnet.ID]; ok { + continue + } + lbZoneSet.Insert(subnet.Zone.Name) + } + + nodeZoneSet := sets.New[string]() + for _, subnet := range clusterNodeSubnets { + // We validate edge subnets in another place. + if _, ok := subnetDataGroups.Edge[subnet.ID]; ok { + continue + } + nodeZoneSet.Insert(subnet.Zone.Name) + } + + // If the nodes use an AZ that is not in load balancer enabled AZs, + // the router pod might be scheduled to nodes that the load balancer cannot reach. + if diffSet := nodeZoneSet.Difference(lbZoneSet); diffSet.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are not enabled for %s load balancers, nodes in those zones are unreachable", sets.List(diffSet), lbType))) + } + + // If the load balancer includes an AZ that is not in node AZs, + // there will be no nodes in that AZ for the load balancer to register (i.e. not in use) + if diffSet := lbZoneSet.Difference(nodeZoneSet); diffSet.Len() > 0 { + allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are enabled for %s load balancers, but are not used by any nodes", sets.List(diffSet), lbType))) + } + + return allErrs +} + func validateServiceEndpoints(fldPath *field.Path, region string, services []awstypes.ServiceEndpoint) field.ErrorList { allErrs := field.ErrorList{} // Validate the endpoint overrides for all provided services. @@ -527,17 +788,13 @@ func validateZoneLocal(ctx context.Context, meta *Metadata, fldPath *field.Path, } func validateEndpointAccessibility(endpointURL string) error { - // For each provided service endpoint, verify we can resolve and connect with net.Dial. - // Ignore e2e.local from unit tests. - if endpointURL == "e2e.local" { - return nil + if _, err := url.Parse(endpointURL); err != nil { + return fmt.Errorf("failed to parse service endpoint url: %w", err) } - _, err := url.Parse(endpointURL) - if err != nil { - return err + if _, err := http.Head(endpointURL); err != nil { //nolint:gosec + return fmt.Errorf("failed to connect to service endpoint url: %w", err) } - _, err = http.Head(endpointURL) - return err + return nil } var requiredServices = []string{ diff --git a/pkg/asset/installconfig/aws/validation_test.go b/pkg/asset/installconfig/aws/validation_test.go index f15ff17ef3..5e0622f1fe 100644 --- a/pkg/asset/installconfig/aws/validation_test.go +++ b/pkg/asset/installconfig/aws/validation_test.go @@ -3,12 +3,15 @@ package aws import ( "context" "fmt" + "net/http" "os" "sort" + "strings" "testing" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/route53" + "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,953 +25,1070 @@ import ( ) var ( - validCIDR = "10.0.0.0/16" - validRegion = "us-east-1" + metaName = "ClusterMetaName" + validCIDR = "10.0.0.0/16" + validVPCID = "vpc-valid-id" + validCallerRef = "valid-caller-reference" validDSId = "valid-delegation-set-id" - validNameServers = []string{"valid-name-server"} - validHostedZoneName = "valid-private-subnet-a" + validHostedZoneName = "valid-hosted-zone" invalidHostedZoneName = "invalid-hosted-zone" - validDomainName = "valid-base-domain" - invalidBaseDomain = "invalid-base-domain" - metaName = "ClusterMetaName" + validNameServers = []string{"valid-name-server"} - publishInternal = func(ic *types.InstallConfig) { ic.Publish = types.InternalPublishingStrategy } - clearHostedZone = func(ic *types.InstallConfig) { ic.AWS.HostedZone = "" } - invalidateHostedZone = func(ic *types.InstallConfig) { ic.AWS.HostedZone = invalidHostedZoneName } - invalidateBaseDomain = func(ic *types.InstallConfig) { ic.BaseDomain = invalidBaseDomain } - clearBaseDomain = func(ic *types.InstallConfig) { ic.BaseDomain = "" } - invalidateRegion = func(ic *types.InstallConfig) { ic.AWS.Region = "us-east4" } + validDomainName = "valid-base-domain" + invalidBaseDomain = "invalid-base-domain" + + // Convert IDs to Subnet type. + subnetsFromIDs = func(ids []string) []aws.Subnet { + subnets := make([]aws.Subnet, len(ids)) + for idx, id := range ids { + subnets[idx] = aws.Subnet{ID: aws.AWSSubnetID(id)} + } + // We sort the by ID to ensure a predictable error output in tests + sort.Slice(subnets, func(i, j int) bool { + return subnets[i].ID < subnets[j].ID + }) + return subnets + } + + // Remove http or https scheme from a URL if any. + trimURLScheme = func(url string) string { + if str, found := strings.CutPrefix(url, "https://"); found { + return str + } + if str, found := strings.CutPrefix(url, "http://"); found { + return str + } + return url + } ) -type editFunctions []func(ic *types.InstallConfig) - -func validInstallConfig() *types.InstallConfig { - return &types.InstallConfig{ - Networking: &types.Networking{ - MachineNetwork: []types.MachineNetworkEntry{ - {CIDR: *ipnet.MustParseCIDR(validCIDR)}, - }, - }, - BaseDomain: validDomainName, - Publish: types.ExternalPublishingStrategy, - Platform: types.Platform{ - AWS: &aws.Platform{ - Region: "us-east-1", - VPC: aws.VPC{ - Subnets: []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - {ID: "valid-public-subnet-a"}, - {ID: "valid-public-subnet-b"}, - {ID: "valid-public-subnet-c"}, - }, - }, - HostedZone: validHostedZoneName, - }, - }, - ControlPlane: &types.MachinePool{ - Architecture: types.ArchitectureAMD64, - Replicas: ptr.To[int64](3), - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "c"}, - }, - }, - }, - Compute: []types.MachinePool{{ - Name: types.MachinePoolComputeRoleName, - Architecture: types.ArchitectureAMD64, - Replicas: ptr.To[int64](3), - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "c"}, - }, - }, - }}, - ObjectMeta: metav1.ObjectMeta{ - Name: metaName, - }, - } -} - -// validInstallConfigEdgeSubnets returns install-config for edge compute pool -// for existing VPC (subnets). -func validInstallConfigEdgeSubnets() *types.InstallConfig { - ic := validInstallConfig() - edgeSubnets := validEdgeSubnets() - for subnet := range edgeSubnets { - ic.Platform.AWS.VPC.Subnets = append(ic.Platform.AWS.VPC.Subnets, aws.Subnet{ID: aws.AWSSubnetID(subnet)}) - } - ic.Compute = append(ic.Compute, types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{}, - }, - }) - return ic -} - -func validAvailZones() []string { - return []string{"a", "b", "c"} -} - -func validAvailRegions() []string { - return []string{"us-east-1", "us-central-1"} -} - -func validAvailZonesWithEdge() []string { - return []string{"a", "b", "c", "edge-a", "edge-b", "edge-c"} -} - -func validAvailZonesOnlyEdge() []string { - return []string{"edge-a", "edge-b", "edge-c"} -} - -func validPrivateSubnets() Subnets { - return Subnets{ - "valid-private-subnet-a": { - Zone: &Zone{Name: "a"}, - CIDR: "10.0.1.0/24", - }, - "valid-private-subnet-b": { - Zone: &Zone{Name: "b"}, - CIDR: "10.0.2.0/24", - }, - "valid-private-subnet-c": { - Zone: &Zone{Name: "c"}, - CIDR: "10.0.3.0/24", - }, - } -} - -func validPublicSubnets() Subnets { - return Subnets{ - "valid-public-subnet-a": { - Zone: &Zone{Name: "a"}, - CIDR: "10.0.4.0/24", - }, - "valid-public-subnet-b": { - Zone: &Zone{Name: "b"}, - CIDR: "10.0.5.0/24", - }, - "valid-public-subnet-c": { - Zone: &Zone{Name: "c"}, - CIDR: "10.0.6.0/24", - }, - } -} - -func validEdgeSubnets() Subnets { - return Subnets{ - "valid-public-subnet-edge-a": { - Zone: &Zone{Name: "edge-a"}, - CIDR: "10.0.7.0/24", - }, - "valid-public-subnet-edge-b": { - Zone: &Zone{Name: "edge-b"}, - CIDR: "10.0.8.0/24", - }, - "valid-public-subnet-edge-c": { - Zone: &Zone{Name: "edge-c"}, - CIDR: "10.0.9.0/24", - }, - } -} - -func validServiceEndpoints() []aws.ServiceEndpoint { - return []aws.ServiceEndpoint{{ - Name: "ec2", - URL: "e2e.local", - }, { - Name: "s3", - URL: "e2e.local", - }, { - Name: "iam", - URL: "e2e.local", - }, { - Name: "elasticloadbalancing", - URL: "e2e.local", - }, { - Name: "tagging", - URL: "e2e.local", - }, { - Name: "route53", - URL: "e2e.local", - }, { - Name: "sts", - URL: "e2e.local", - }} -} - -func invalidServiceEndpoint() []aws.ServiceEndpoint { - return []aws.ServiceEndpoint{{ - Name: "testing", - URL: "testing", - }, { - Name: "test", - URL: "http://testing.non", - }} -} - -func validInstanceTypes() map[string]InstanceType { - return map[string]InstanceType{ - "t2.small": { - DefaultVCpus: 1, - MemInMiB: 2048, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m5.large": { - DefaultVCpus: 2, - MemInMiB: 8192, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m5.xlarge": { - DefaultVCpus: 4, - MemInMiB: 16384, - Arches: []string{ec2.ArchitectureTypeX8664}, - }, - "m6g.xlarge": { - DefaultVCpus: 4, - MemInMiB: 16384, - Arches: []string{ec2.ArchitectureTypeArm64}, - }, - } -} - -func createBaseDomainHostedZone() route53.HostedZone { - return route53.HostedZone{ - CallerReference: &validCallerRef, - Id: &validDSId, - Name: &validDomainName, - } -} - -func createValidHostedZone() route53.GetHostedZoneOutput { - ptrValidNameServers := []*string{} - for i := range validNameServers { - ptrValidNameServers = append(ptrValidNameServers, &validNameServers[i]) - } - - validDelegationSet := route53.DelegationSet{CallerReference: &validCallerRef, Id: &validDSId, NameServers: ptrValidNameServers} - validHostedZone := route53.HostedZone{CallerReference: &validCallerRef, Id: &validDSId, Name: &validHostedZoneName} - validVPCs := []*route53.VPC{{VPCId: &validHostedZoneName, VPCRegion: &validRegion}} - - return route53.GetHostedZoneOutput{ - DelegationSet: &validDelegationSet, - HostedZone: &validHostedZone, - VPCs: validVPCs, - } -} - func TestValidate(t *testing.T) { tests := []struct { - name string - installConfig *types.InstallConfig - availZones []string - availRegions []string - edgeZones []string - privateSubnets Subnets - publicSubnets Subnets - edgeSubnets Subnets - instanceTypes map[string]InstanceType - proxy string - publicOnly string - expectErr string - }{{ - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = nil - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid no byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: validInstallConfig(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: validInstallConfigEdgeSubnets(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid byo", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - availRegions: validAvailRegions(), - }, { - name: "valid instance types", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{ - Region: "us-east-1", - DefaultMachinePlatform: &aws.MachinePool{ - InstanceType: "m5.xlarge", - }, - } - c.ControlPlane.Platform.AWS.InstanceType = "m5.xlarge" - c.Compute[0].Platform.AWS.InstanceType = "m5.large" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - }, { - name: "invalid control plane instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.ControlPlane.Platform.AWS.InstanceType = "t2.small" - c.Compute[0].Platform.AWS.InstanceType = "m5.large" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Q[controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 4 vCPUs, controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 16384 MiB Memory]\E$`, - }, { - name: "invalid compute instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.ControlPlane.Platform.AWS.InstanceType = "m5.xlarge" - c.Compute[0].Platform.AWS.InstanceType = "t2.small" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Q[compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 2 vCPUs, compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 8192 MiB Memory]\E$`, - }, { - name: "undefined compute instance type", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{Region: "us-east-1"} - c.Compute[0].Platform.AWS.InstanceType = "m5.dummy" - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\Qcompute[0].platform.aws.type: Invalid value: "m5.dummy": instance type m5.dummy not found\E$`, - }, { - name: "mismatched instance architecture", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS = &aws.Platform{ - Region: "us-east-1", - DefaultMachinePlatform: &aws.MachinePool{InstanceType: "m5.xlarge"}, - } - c.ControlPlane.Architecture = types.ArchitectureARM64 - c.Compute[0].Platform.AWS.InstanceType = "m6g.xlarge" - c.Compute[0].Architecture = types.ArchitectureAMD64 - return c - }(), - availZones: validAvailZones(), - instanceTypes: validInstanceTypes(), - availRegions: validAvailRegions(), - expectErr: `^\[controlPlane.platform.aws.type: Invalid value: "m5.xlarge": instance type supported architectures \[amd64\] do not match specified architecture arm64, compute\[0\].platform.aws.type: Invalid value: "m6g.xlarge": instance type supported architectures \[arm64\] do not match specified architecture amd64\]$`, - }, { - name: "mismatched compute pools architectures", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[0].Architecture = types.ArchitectureAMD64 - c.Compute[1].Architecture = types.ArchitectureARM64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, - }, { - name: "valid compute pools architectures", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[0].Architecture = types.ArchitectureAMD64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - }, { - name: "mismatched compute pools architectures 2", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Compute[1].Architecture = types.ArchitectureARM64 - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - edgeSubnets: validEdgeSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, - }, { - name: "invalid no private subnets", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-public-subnet-a"}, - {ID: "valid-public-subnet-b"}, - {ID: "valid-public-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - publicSubnets: validPublicSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid no public subnets", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{ - {ID: "valid-private-subnet-a"}, - {ID: "valid-private-subnet-b"}, - {ID: "valid-private-subnet-c"}, - } - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid cidr does not belong to machine CIDR", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "invalid-cidr-subnet"}) - return c - }(), - availZones: func() []string { - zones := validAvailZones() - return append(zones, "zone-for-invalid-cidr-subnet") - }(), - privateSubnets: validPrivateSubnets(), - availRegions: validAvailRegions(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["invalid-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.126.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks$`, - }, { - name: "invalid cidr does not belong to machine CIDR", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "invalid-private-cidr-subnet"}, aws.Subnet{ID: "invalid-public-cidr-subnet"}) - return c - }(), - availZones: func() []string { - zones := validAvailZones() - return append(zones, "zone-for-invalid-cidr-subnet") - }(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["invalid-private-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.126.0/24", - } - return s - }(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["invalid-public-cidr-subnet"] = Subnet{ - Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, - CIDR: "192.168.127.0/24", - } - return s - }(), - expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-private-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks, platform\.aws\.vpc\.subnets\[7\]: Invalid value: \"invalid-public-cidr-subnet\": subnet's CIDR range start 192\.168\.127\.0 is outside of the specified machine networks\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid missing public subnet in a zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "no-matching-public-private-zone"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["no-matching-public-private-zone"] = Subnet{ - Zone: &Zone{Name: "f"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - publicSubnets: validPublicSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"no-matching-public-private-zone\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[f\]$`, - }, { - name: "invalid multiple private in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-private-zone-c-2"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: func() Subnets { - s := validPrivateSubnets() - s["valid-private-zone-c-2"] = Subnet{ - Zone: &Zone{Name: "c"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - publicSubnets: validPublicSubnets(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-private-zone-c-2\": private subnet valid-private-subnet-c is also in zone c$`, - }, { - name: "invalid multiple public in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-public-zone-c-2"}) - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - privateSubnets: validPrivateSubnets(), - publicSubnets: func() Subnets { - s := validPublicSubnets() - s["valid-public-zone-c-2"] = Subnet{ - Zone: &Zone{Name: "c"}, - CIDR: "10.0.7.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-public-zone-c-2\": public subnet valid-public-subnet-c is also in zone c$`, - }, { - name: "invalid multiple public edge in same zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: "valid-public-zone-edge-c-2"}) - return c - }(), - availZones: validAvailZonesWithEdge(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - edgeSubnets: func() Subnets { - s := validEdgeSubnets() - s["valid-public-zone-edge-c-2"] = Subnet{ - Zone: &Zone{Name: "edge-c", Type: aws.LocalZoneType}, - CIDR: "10.0.9.0/24", - } - return s - }(), - expectErr: `^platform\.aws\.vpc\.subnets\[9\]: Invalid value: \"valid-public-zone-edge-c-2\": edge subnet valid-public-subnet-edge-c is also in zone edge-c$`, - }, { - name: "invalid edge pool missing valid subnets", - installConfig: validInstallConfigEdgeSubnets(), - availZones: validAvailZonesWithEdge(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - edgeSubnets: Subnets{}, - expectErr: `^compute\[1\]\.platform\.aws: Required value: the provided subnets must include valid subnets for the specified edge zones$`, - }, { - name: "invalid edge pool missing zones", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{}, - }, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, - }, { - name: "invalid edge pool empty zones", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{}, + name string + installConfig *types.InstallConfig + availRegions []string + availZones []string + edgeZones []string + subnets SubnetGroups + subnetsInVPC *SubnetGroups + instanceTypes map[string]InstanceType + proxy string + publicOnly bool + expectErr string + }{ + { + name: "valid instance types", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "m5.large")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + }, + { + name: "invalid control plane instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "t2.small", "m5.large")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Q[controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 4 vCPUs, controlPlane.platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 16384 MiB Memory]\E$`, + }, + { + name: "invalid compute instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "t2.small")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Q[compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 2 vCPUs, compute[0].platform.aws.type: Invalid value: "t2.small": instance type does not meet minimum resource requirements of 8192 MiB Memory]\E$`, + }, + { + name: "invalid undefined compute instance type", + installConfig: icBuild.build(icBuild.withInstanceType("m5.xlarge", "m5.xlarge", "m5.dummy")), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\Qcompute[0].platform.aws.type: Invalid value: "m5.dummy": instance type m5.dummy not found\E$`, + }, + { + name: "valid compute pools architectures", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64, types.ArchitectureAMD64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "invalid mismatched instance architecture", + installConfig: icBuild.build( + icBuild.withInstanceType("m5.xlarge", "", "m6g.xlarge"), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + expectErr: `^\[controlPlane.platform.aws.type: Invalid value: "m5.xlarge": instance type supported architectures \[amd64\] do not match specified architecture arm64, compute\[0\].platform.aws.type: Invalid value: "m6g.xlarge": instance type supported architectures \[arm64\] do not match specified architecture amd64\]$`, + }, + { + name: "invalid mismatched compute pools architectures", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withInstanceArchitecture(types.ArchitectureARM64, types.ArchitectureAMD64, types.ArchitectureARM64), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + instanceTypes: validInstanceTypes(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `^compute\[1\].architecture: Invalid value: "arm64": all compute machine pools must be of the same architecture$`, + }, + { + name: "invalid edge pool, missing zones", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, }, - }, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, - }, { - name: "invalid edge pool missing platform definition", - installConfig: func() *types.InstallConfig { - ic := validInstallConfig() - ic.Platform.AWS.VPC.Subnets = []aws.Subnet{} - ic.ControlPlane = &types.MachinePool{} - edgePool := types.MachinePool{ - Name: types.MachinePoolEdgeRoleName, - Platform: types.MachinePoolPlatform{}, - } - ic.Compute = []types.MachinePool{edgePool} - return ic - }(), - availRegions: validAvailRegions(), - expectErr: `^\[compute\[0\]\.platform\.aws: Required value: edge compute pools are only supported on the AWS platform, compute\[0\].platform.aws: Required value: zone is required when using edge machine pools\]$`, - }, { - name: "invalid edge pool missing subnets on availability zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfigEdgeSubnets() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - edgeSubnets := validEdgeSubnets() - for subnet := range edgeSubnets { - c.Platform.AWS.VPC.Subnets = append(c.Platform.AWS.VPC.Subnets, aws.Subnet{ID: aws.AWSSubnetID(subnet)}) - } - sort.Slice(c.Platform.AWS.VPC.Subnets, func(i, j int) bool { - subnets := c.Platform.AWS.VPC.Subnets - return subnets[i].ID < subnets[j].ID - }) - return c - }(), - availZones: validAvailZonesOnlyEdge(), - privateSubnets: Subnets{}, - publicSubnets: Subnets{}, - edgeSubnets: validEdgeSubnets(), - expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-public-subnet-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, - availRegions: validAvailRegions(), - }, { - name: "invalid no subnet for control plane zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.ControlPlane.Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, - }, { - name: "invalid no subnet for control plane zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.ControlPlane.Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d", "e") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\", \"e\"}: No subnets provided for zones \[d e\]$`, - }, { - name: "invalid no subnet for compute[0] zones", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Compute[0].Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, - }, { - name: "invalid no subnet for compute zone", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Compute[0].Platform.AWS.Zones = append(c.ControlPlane.Platform.AWS.Zones, "d") - c.Compute = append(c.Compute, types.MachinePool{ - Architecture: types.ArchitectureAMD64, - Platform: types.MachinePoolPlatform{ - AWS: &aws.MachinePool{ - Zones: []string{"a", "b", "e"}, + }}, true), + icBuild.withControlPlaneMachinePool(types.MachinePool{}), + ), + availRegions: validAvailRegions(), + expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, + }, + { + name: "invalid edge pool, empty zones", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{}, + }, }, - }, - }) - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^\[compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\], compute\[1\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"e\"}: No subnets provided for zones \[e\]\]$`, - }, { - name: "custom region invalid service endpoints none provided", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "custom region invalid service endpoints some provided", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints()[:3] - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "custom region valid service endpoints", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.AMIID = "dummy-id" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI omitted for new region in standard partition", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-newregion-1" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: "platform.aws.amiID: Required value: AMI must be provided", - }, { - name: "accept platform-level AMI", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "accept AMI from default machine platform", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.Platform.AWS.DefaultMachinePlatform = &aws.MachinePool{AMIID: "custom-ami"} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "accept AMIs specified for each machine pool", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.ControlPlane.Platform.AWS.AMIID = "custom-ami" - c.Compute[0].Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI omitted for compute with no replicas", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-gov-east-1" - c.ControlPlane.Platform.AWS.AMIID = "custom-ami" - c.Compute[0].Replicas = ptr.To[int64](0) - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - }, { - name: "AMI not provided for unknown region", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "test-region" - c.Platform.AWS.ServiceEndpoints = validServiceEndpoints() - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^platform\.aws\.amiID: Required value: AMI must be provided$`, - }, { - name: "invalid endpoint URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.ServiceEndpoints = invalidServiceEndpoint() - c.Platform.AWS.AMIID = "custom-ami" - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "testing": Head "testing": unsupported protocol scheme "", platform.aws.serviceEndpoints[1].url: Invalid value: "http://testing.non": Head "http://testing.non": dial tcp: lookup testing.non\E.*: no such host\]$`, - }, { - name: "invalid proxy URL but valid URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.AMIID = "custom-ami" - c.Platform.AWS.ServiceEndpoints = []aws.ServiceEndpoint{{Name: "test", URL: "http://testing.com"}} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - proxy: "proxy", - }, { - name: "invalid proxy URL and invalid URL", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.Region = "us-east-1" - c.Platform.AWS.AMIID = "custom-ami" - c.Platform.AWS.ServiceEndpoints = []aws.ServiceEndpoint{{Name: "test", URL: "http://test"}} - return c - }(), - availZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - proxy: "http://proxy.com", - expectErr: `^\Qplatform.aws.serviceEndpoints[0].url: Invalid value: "http://test": Head "http://test": dial tcp: lookup test\E.*: no such host$`, - }, { - name: "invalid public ipv4 pool private installation", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - c.Platform.AWS.PublicIpv4Pool = "ipv4pool-ec2-123" - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - expectErr: `^platform.aws.publicIpv4PoolId: Invalid value: "ipv4pool-ec2-123": publish strategy Internal can't be used with custom Public IPv4 Pools$`, - }, { - name: "invalid publish method for public-only subnets install", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Publish = types.InternalPublishingStrategy - return c - }(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availRegions: validAvailRegions(), - publicOnly: "true", - expectErr: `^publish: Invalid value: \"Internal\": cluster cannot be private with public subnets$`, - }, { - name: "no public subnets specified for public-only subnets cluster", - installConfig: validInstallConfig(), - privateSubnets: validPrivateSubnets(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - expectErr: `^\[platform\.aws\.vpc\.subnets: Required value: public subnets are required for a public-only subnets cluster, platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"valid-private-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-private-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"valid-public-subnet-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]\]$`, - }, { - name: "no subnets specified for public-only subnets cluster", - installConfig: func() *types.InstallConfig { - c := validInstallConfig() - c.Platform.AWS.VPC.Subnets = []aws.Subnet{} - return c - }(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - }, { - name: "valid public-only subnets install config", - installConfig: validInstallConfig(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - availZones: validAvailZones(), - availRegions: validAvailRegions(), - publicOnly: "true", - }} + }}, true), + ), + availRegions: validAvailRegions(), + expectErr: `^compute\[0\]\.platform\.aws: Required value: zone is required when using edge machine pools$`, + }, + { + name: "invalid edge pool missing platform definition", + installConfig: icBuild.build( + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{}, + }}, true), + icBuild.withControlPlaneMachinePool(types.MachinePool{}), + ), + availRegions: validAvailRegions(), + expectErr: `^\[compute\[0\]\.platform\.aws: Required value: edge compute pools are only supported on the AWS platform, compute\[0\].platform.aws: Required value: zone is required when using edge machine pools\]$`, + }, + { + name: "valid service endpoints, custom region and no endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + ), + }, + { + name: "valid service endpoints, custom region and some endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + icBuild.withServiceEndpoints(validServiceEndpoints()[:3], true), + ), + }, + { + name: "valid service endpoints, custom region and all endpoints provided", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + icBuild.withPlatformAMIID("dummy-id"), + icBuild.withServiceEndpoints(validServiceEndpoints(), true), + ), + }, + { + name: "invalid service endpoint URLs", + installConfig: icBuild.build( + icBuild.withServiceEndpoints(invalidServiceEndpoint(), true), + ), + availRegions: validAvailRegions(), + expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "bad-aws-endpoint": failed to connect to service endpoint url: Head "bad-aws-endpoint": dial tcp: lookup bad-aws-endpoint: no such host, platform.aws.serviceEndpoints[1].url: Invalid value: "http://bad-aws-endpoint.non": failed to connect to service endpoint url: Head "http://bad-aws-endpoint.non": dial tcp: lookup bad-aws-endpoint.non: no such host]\E$`, + }, + { + name: "valid AMI, from platform level", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withPlatformAMIID("custom-ami"), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMI, from default platform machine", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withDefaultPlatformMachine(aws.MachinePool{AMIID: "custom-ami"}), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMIs, from machine pools", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withControlPlanePlatformAMI("custom-ami"), + icBuild.withComputePlatformAMI("custom-ami", 0), + ), + availRegions: validAvailRegions(), + }, + { + name: "valid AMI, omitted for compute with no replicas", + installConfig: icBuild.build( + icBuild.withPlatformRegion("us-gov-east-1"), + icBuild.withControlPlanePlatformAMI("custom-ami"), + icBuild.withComputeReplicas(0, 0), + ), + availRegions: validAvailRegions(), + }, + { + name: "invalid AMI not provided for unknown region", + installConfig: icBuild.build( + icBuild.withPlatformRegion("test-region"), + ), + availRegions: validAvailRegions(), + expectErr: `^platform\.aws\.amiID: Required value: AMI must be provided$`, + }, + { + name: "invalid proxy URL but valid service endpoint URL", + installConfig: icBuild.build( + icBuild.withPlatformAMIID("custom-ami"), + icBuild.withServiceEndpoints(validServiceEndpoints(), true), + ), + availRegions: validAvailRegions(), + proxy: "proxy", + }, + { + name: "invalid proxy URL and invalid service endpoint URL", + installConfig: icBuild.build( + icBuild.withPlatformAMIID("custom-ami"), + icBuild.withServiceEndpoints(invalidServiceEndpoint(), true), + ), + availRegions: validAvailRegions(), + proxy: "http://proxy.com", + expectErr: `^\Q[platform.aws.serviceEndpoints[0].url: Invalid value: "bad-aws-endpoint": failed to connect to service endpoint url: Head "bad-aws-endpoint": dial tcp: lookup bad-aws-endpoint: no such host, platform.aws.serviceEndpoints[1].url: Invalid value: "http://bad-aws-endpoint.non": failed to connect to service endpoint url: Head "http://bad-aws-endpoint.non": dial tcp: lookup bad-aws-endpoint.non: no such host]\E$`, + }, + { + name: "invalid public ipv4 pool in private installation", + installConfig: icBuild.build( + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withPublicIPv4Pool("ipv4pool-ec2-123"), + icBuild.withVPCSubnets([]aws.Subnet{}, true), + ), + availRegions: validAvailRegions(), + expectErr: `^platform.aws.publicIpv4PoolId: Invalid value: "ipv4pool-ec2-123": publish strategy Internal can't be used with custom Public IPv4 Pools$`, + }, + { + name: "valid no byo subnets, unspecified subnet list", + installConfig: icBuild.build(), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + }, + { + name: "valid no byo subnets, empty subnet list", + installConfig: icBuild.build(icBuild.withVPCSubnets([]aws.Subnet{}, true)), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + }, + { + name: "valid no byo subnets, unspecified subnet list for public-only subnets cluster", + installConfig: icBuild.build(), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + publicOnly: true, + }, + { + name: "valid byo subnets", + installConfig: icBuild.build(icBuild.withBaseBYO()), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets, include edge subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets, private subnets only with publish internal", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets, no private subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnetIDs(validSubnets("public").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-public-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + }, + { + name: "invalid byo subnets, no public subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-private-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[a b c\]$`, + }, + { + name: "invalid byo subnets, invalid cidr does not belong to machine CIDR", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"invalid-private-cidr-subnet", "invalid-public-cidr-subnet"}, false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), "zone-for-invalid-cidr-subnet"), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"invalid-private-cidr-subnet": Subnet{ + ID: "invalid-private-cidr-subnet", + Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, + CIDR: "192.168.126.0/24", + }}), + Public: mergeSubnets(validSubnets("public"), Subnets{"invalid-public-cidr-subnet": Subnet{ + ID: "invalid-public-cidr-subnet", + Zone: &Zone{Name: "zone-for-invalid-cidr-subnet"}, + CIDR: "192.168.127.0/24", + }}), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"invalid-private-cidr-subnet\": subnet's CIDR range start 192\.168\.126\.0 is outside of the specified machine networks, platform\.aws\.vpc\.subnets\[7\]: Invalid value: \"invalid-public-cidr-subnet\": subnet's CIDR range start 192\.168\.127\.0 is outside of the specified machine networks\]$`, + }, + { + name: "invalid byo subnets, missing public subnet in a zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"no-matching-public-private-zone"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"no-matching-public-private-zone": Subnet{ + ID: "no-matching-public-private-zone", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.7.0/24", + }}), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-private-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-private-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"no-matching-public-private-zone\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: No public subnet provided for zones \[f\]$`, + }, + { + name: "invalid byo subnets, multiple private in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"valid-private-zone-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{"valid-private-zone-c-2": Subnet{ + ID: "valid-private-zone-c-2", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.7.0/24", + }}), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-private-zone-c-2\": private subnet subnet-valid-private-c is also in zone c$`, + }, + { + name: "invalid byo subnets, multiple public in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs([]string{"valid-public-zone-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{"valid-public-zone-c-2": Subnet{ + ID: "valid-public-zone-c-2", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.7.0/24", + }}), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"valid-public-zone-c-2\": public subnet subnet-valid-public-c is also in zone c$`, + }, + { + name: "invalid byo subnets, multiple public edge in same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + icBuild.withVPCEdgeSubnetIDs([]string{"valid-public-zone-edge-c-2"}, false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), validEdgeAvailZones()...), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: mergeSubnets(validSubnets("edge"), Subnets{ + "valid-public-zone-edge-c-2": Subnet{ + ID: "valid-public-zone-edge-c-2", + Zone: &Zone{Name: "edge-c", Type: aws.LocalZoneType}, + CIDR: "10.0.9.0/24", + }, + }), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets\[9\]: Invalid value: \"valid-public-zone-edge-c-2\": edge subnet subnet-valid-public-edge-c is also in zone edge-c$`, + }, + { + name: "invalid byo subnets, edge pool missing valid subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), false), + ), + availRegions: validAvailRegions(), + availZones: append(validAvailZones(), validEdgeAvailZones()...), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^compute\[1\]\.platform\.aws: Required value: the provided subnets must include valid subnets for the specified edge zones$`, + }, + { + name: "invalid byo subnets, edge pool missing subnets on availability zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCEdgeSubnetIDs(validSubnets("edge").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validEdgeAvailZones(), + subnets: SubnetGroups{ + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets: Invalid value: \[\]aws\.Subnet\{aws\.Subnet\{ID:\"subnet-valid-public-edge-a\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-edge-b\", Roles:\[\]aws\.SubnetRole\(nil\)\}, aws\.Subnet\{ID:\"subnet-valid-public-edge-c\", Roles:\[\]aws\.SubnetRole\(nil\)\}\}: no private subnets found, controlPlane\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\], compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string\{\"a\", \"b\", \"c\"\}: No subnets provided for zones \[a b c\]\]$`, + }, + { + name: "invalid byo subnets, no subnet for control plane zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withControlPlanePlatformZones([]string{"d", "e"}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^controlPlane\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\", \"e\"}: No subnets provided for zones \[d e\]$`, + }, + { + name: "invalid byo subnets, no subnet for compute[0] zones", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withComputePlatformZones([]string{"d"}, false, 0), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\]$`, + }, + { + name: "invalid byo subnets, no subnet for compute zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withComputePlatformZones([]string{"d"}, false, 0), + icBuild.withComputeMachinePool([]types.MachinePool{{ + Architecture: types.ArchitectureAMD64, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{ + Zones: []string{"a", "b", "e"}, + }, + }, + }}, false), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[compute\[0\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"c\", \"d\"}: No subnets provided for zones \[d\], compute\[1\]\.platform\.aws\.zones: Invalid value: \[\]string{\"a\", \"b\", \"e\"}: No subnets provided for zones \[e\]\]$`, + }, + { + name: "valid byo subnets, private and public subnets provided for public-only subnets cluster", + installConfig: icBuild.build(icBuild.withBaseBYO()), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("public"), validSubnets("private")), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "valid byo subnets, public subnets provided for public-only subnets cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("public").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("public"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "invalid byo subnets, no public subnets specified for public-only subnets cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnetIDs(validSubnets("private").IDs(), true), + ), + availRegions: validAvailRegions(), + availZones: validAvailZones(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + VPC: validVPCID, + }, + publicOnly: true, + expectErr: `^\Q[platform.aws.vpc.subnets: Required value: public subnets are required for a public-only subnets cluster, platform.aws.vpc.subnets: Invalid value: []aws.Subnet{aws.Subnet{ID:"subnet-valid-private-a", Roles:[]aws.SubnetRole(nil)}, aws.Subnet{ID:"subnet-valid-private-b", Roles:[]aws.SubnetRole(nil)}, aws.Subnet{ID:"subnet-valid-private-c", Roles:[]aws.SubnetRole(nil)}}: No public subnet provided for zones [a b c]]\E$`, + }, + { + name: "invalid byo subnets, internal publish method for public-only subnets install", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + expectErr: `^publish: Invalid value: \"Internal\": cluster cannot be private with public subnets$`, + }, + { + name: "valid byo subnets, no roles and vpc has no untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherTaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets, no roles but vpc has untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherUntaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^platform\.aws\.vpc\.subnets: Forbidden: additional subnets \[subnet-valid-private-a1 subnet-valid-private-b1\] without tag prefix kubernetes\.io/cluster/ are found in vpc vpc-valid-id of provided subnets\. Please add a tag kubernetes\.io/cluster/unmanaged to those subnets to exclude them from cluster installation or explicitly assign roles in the install-config to provided subnets$`, + }, + { + name: "valid byo subnets with roles", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, include edge subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCEdgeSubnets(byoEdgeSubnetsWithRoles(), false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, ignore other untagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherUntaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "valid byo subnets with roles, vpc has other tagged subnets", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + subnetsInVPC: &SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), otherTaggedPrivateSubnets()), + Public: validSubnets("public"), + VPC: validVPCID, + }, + }, + { + name: "invalid byo subnets with roles, public subnet assigned ClusterNode", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.ClusterNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-public-a1": public subnet subnet-valid-public-a is also in zone a, platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-public-a1": subnet subnet-valid-public-a1 has role ClusterNode, but is public, expected to be private]\E$`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned Bootstrap in external cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-private-a1": private subnet subnet-valid-private-a is also in zone a, platform.aws.vpc.subnets[6]: Invalid value: "subnet-valid-private-a1": subnet subnet-valid-private-a1 has role BootstrapNode, but is private, expected to be public]\E$`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned ControlPlaneExternalLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneExternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": private subnet subnet-valid-private-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": subnet subnet-valid-private-a1 has role ControlPlaneExternalLB, but is private, expected to be public\]$`, + }, + { + name: "invalid byo subnets with roles, public subnet assigned ControlPlaneInternalLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneInternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\[platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": public subnet subnet-valid-public-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnet subnet-valid-public-a1 has role ControlPlaneInternalLB, but is public, expected to be private\]$`, + }, + { + name: "valid byo subnets with roles, public subnet assigned ControlPlaneInternalLB, ClusterNode and public-only cluster", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoPublicOnlySubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Public: validSubnets("public"), + Private: validSubnets("public"), + VPC: validVPCID, + }, + publicOnly: true, + }, + { + name: "invalid byo subnets with roles, public subnet assigned IngressControllerLB when publish is internal", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withPublish(types.InternalPublishingStrategy), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[3\]: Invalid value: \"subnet-valid-public-a\": subnet subnet-valid-public-a has role IngressControllerLB and is public, which is not allowed when publish is set to Internal`, + }, + { + name: "invalid byo subnets with roles, private subnet assigned IngressControllerLB when publish is external", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-a1", + Roles: []aws.SubnetRole{{Type: aws.IngressControllerLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: validSubnets("public"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-private-a1\": subnet subnet-valid-private-a1 has role IngressControllerLB and is private, which is not allowed when publish is set to External`, + }, + { + name: "invalid byo subnets with roles, subnets assigned IngressControllerLB in the same zone", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.IngressControllerLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": public subnet subnet-valid-public-a is also in zone a, platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnets subnet-valid-public-a and subnet-valid-public-a1 have role IngressControllerLB and are both in zone a`, + }, + { + name: "invalid byo subnets with roles, AZs of IngressControllerLB and ControlPlaneLB not match AZs of ClusterNode", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-f", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneExternalLBSubnetRole}, {Type: aws.IngressControllerLBSubnetRole}}, + }, + { + ID: "subnet-valid-private-f", + Roles: []aws.SubnetRole{{Type: aws.ControlPlaneInternalLBSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-f": { + ID: "subnet-valid-private-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-f": { + ID: "subnet-valid-public-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for ControlPlaneInternalLB load balancers, but are not used by any nodes, platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for IngressControllerLB load balancers, but are not used by any nodes, platform.aws.vpc.subnets: Forbidden: zones [f] are enabled for ControlPlaneExternalLB load balancers, but are not used by any nodes]\E$`, + }, + { + name: "invalid byo subnets with roles, AZs of ClusterNode not match AZs of IngressControllerLB and ControlPlaneLB", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-private-f", + Roles: []aws.SubnetRole{{Type: aws.ClusterNodeSubnetRole}}, + }, + { + ID: "subnet-valid-public-f", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: mergeSubnets(validSubnets("private"), Subnets{ + "subnet-valid-private-f": { + ID: "subnet-valid-private-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + }, + }), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-f": { + ID: "subnet-valid-public-f", + Zone: &Zone{Name: "f"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `^\Q[platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for ControlPlaneInternalLB load balancers, nodes in those zones are unreachable, platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for IngressControllerLB load balancers, nodes in those zones are unreachable, platform.aws.vpc.subnets: Forbidden: zones [f] are not enabled for ControlPlaneExternalLB load balancers, nodes in those zones are unreachable]\E$`, + }, + { + name: "invalid byo subnets with roles, subnet assigned EdgeNode but not edge subnet", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCEdgeSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-a1", + Roles: []aws.SubnetRole{{Type: aws.EdgeNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: mergeSubnets(validSubnets("public"), Subnets{ + "subnet-valid-public-a1": { + ID: "subnet-valid-public-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + Edge: validSubnets("edge"), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-a1\": subnet subnet-valid-public-a1 has role EdgeNode, but is not in a Local or WaveLength Zone`, + }, + { + name: "invalid byo subnets with roles, edge subnet assigned with other roles", + installConfig: icBuild.build( + icBuild.withBaseBYO(), + icBuild.withVPCSubnets(byoSubnetsWithRoles(), true), + icBuild.withVPCSubnets([]aws.Subnet{ + { + ID: "subnet-valid-public-edge-a1", + Roles: []aws.SubnetRole{{Type: aws.BootstrapNodeSubnetRole}}, + }, + }, false), + ), + availRegions: validAvailRegions(), + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + Edge: mergeSubnets(validSubnets("edge"), Subnets{ + "subnet-valid-public-edge-a1": { + ID: "subnet-valid-public-edge-a1", + Zone: &Zone{Name: "edge-a"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + }), + VPC: validVPCID, + }, + expectErr: `platform\.aws\.vpc\.subnets\[6\]: Invalid value: \"subnet-valid-public-edge-a1\": subnet subnet-valid-public-edge-a1 must only be assigned role EdgeNode since it is in a Local or WaveLength Zone`, + }, + } + + // Register mock http(s) responses for tests. + httpmock.Activate() + t.Cleanup(httpmock.Deactivate) + + for _, endpoint := range validServiceEndpoints() { + httpmock.RegisterResponder(http.MethodHead, endpoint.URL, func(r *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(http.StatusOK, ""), nil + }) + } + + for _, endpoint := range invalidServiceEndpoint() { + httpmock.RegisterResponder(http.MethodHead, endpoint.URL, func(r *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("dial tcp: lookup %s: no such host", trimURLScheme(endpoint.URL)) + }) + } for _, test := range tests { t.Run(test.name, func(t *testing.T) { meta := &Metadata{ availabilityZones: test.availZones, availableRegions: test.availRegions, - privateSubnets: test.privateSubnets, - publicSubnets: test.publicSubnets, - edgeSubnets: test.edgeSubnets, + edgeZones: test.edgeZones, + subnets: test.subnets, + vpcSubnets: test.subnets, + vpc: validVPCID, instanceTypes: test.instanceTypes, - Subnets: test.installConfig.Platform.AWS.VPC.Subnets, + ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets, + } + + if test.subnetsInVPC != nil { + meta.vpcSubnets = *test.subnetsInVPC } if test.proxy != "" { os.Setenv("HTTP_PROXY", test.proxy) } else { os.Unsetenv("HTTP_PROXY") } - if test.publicOnly != "" { - os.Setenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY", test.publicOnly) + if test.publicOnly { + os.Setenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY", "true") } else { os.Unsetenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY") } + err := Validate(context.TODO(), meta, test.installConfig) if test.expectErr == "" { assert.NoError(t, err) @@ -1030,35 +1150,35 @@ func TestIsHostedZoneDomainParentOfClusterDomain(t *testing.T) { func TestValidateForProvisioning(t *testing.T) { cases := []struct { name string - edits editFunctions + icOptions []icOption expectedErr string }{{ // This really should test for nil, as nothing happened, but no errors were provided - name: "internal publish strategy no hosted zone", - edits: editFunctions{publishInternal, clearHostedZone}, + name: "internal publish strategy no hosted zone", + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy), icBuild.withHostedZone("")}, }, { name: "external publish strategy no hosted zone invalid (empty) base domain", - edits: editFunctions{clearHostedZone, clearBaseDomain}, + icOptions: []icOption{icBuild.withHostedZone(""), icBuild.withBaseDomain("")}, expectedErr: "baseDomain: Invalid value: \"\": cannot find base domain", }, { name: "external publish strategy no hosted zone invalid base domain", - edits: editFunctions{clearHostedZone, invalidateBaseDomain}, + icOptions: []icOption{icBuild.withHostedZone(""), icBuild.withBaseDomain(invalidBaseDomain)}, expectedErr: "baseDomain: Invalid value: \"invalid-base-domain\": cannot find base domain", }, { - name: "external publish strategy no hosted zone valid base domain", - edits: editFunctions{clearHostedZone}, + name: "external publish strategy no hosted zone valid base domain", + icOptions: []icOption{icBuild.withHostedZone("")}, }, { - name: "internal publish strategy valid hosted zone", - edits: editFunctions{publishInternal}, + name: "internal publish strategy valid hosted zone", + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy)}, }, { name: "internal publish strategy invalid hosted zone", - edits: editFunctions{publishInternal, invalidateHostedZone}, + icOptions: []icOption{icBuild.withPublish(types.InternalPublishingStrategy), icBuild.withHostedZone(invalidHostedZoneName)}, expectedErr: "aws.hostedZone: Invalid value: \"invalid-hosted-zone\": unable to retrieve hosted zone", }, { name: "external publish strategy valid hosted zone", }, { name: "external publish strategy invalid hosted zone", - edits: editFunctions{invalidateHostedZone}, + icOptions: []icOption{icBuild.withHostedZone(invalidHostedZoneName)}, expectedErr: "aws.hostedZone: Invalid value: \"invalid-hosted-zone\": unable to retrieve hosted zone", }} @@ -1067,8 +1187,8 @@ func TestValidateForProvisioning(t *testing.T) { route53Client := mock.NewMockAPI(mockCtrl) - validHostedZoneOutput := createValidHostedZone() - validDomainOutput := createBaseDomainHostedZone() + validHostedZoneOutput := createValidHostedZoneOutput() + validDomainOutput := createBaseDomainHostedZoneOutput() route53Client.EXPECT().GetBaseDomain(validDomainName).Return(&validDomainOutput, nil).AnyTimes() route53Client.EXPECT().GetBaseDomain("").Return(nil, fmt.Errorf("invalid value: \"\": cannot find base domain")).AnyTimes() @@ -1083,22 +1203,21 @@ func TestValidateForProvisioning(t *testing.T) { for _, test := range cases { t.Run(test.name, func(t *testing.T) { - editedInstallConfig := validInstallConfig() - for _, edit := range test.edits { - edit(editedInstallConfig) - } - + ic := icBuild.build(test.icOptions...) meta := &Metadata{ availabilityZones: validAvailZones(), - privateSubnets: validPrivateSubnets(), - publicSubnets: validPublicSubnets(), - instanceTypes: validInstanceTypes(), - Region: editedInstallConfig.AWS.Region, - vpc: "valid-private-subnet-a", - Subnets: editedInstallConfig.Platform.AWS.VPC.Subnets, + subnets: SubnetGroups{ + Private: validSubnets("private"), + Public: validSubnets("public"), + VPC: validVPCID, + }, + instanceTypes: validInstanceTypes(), + Region: ic.AWS.Region, + vpc: validVPCID, + ProvidedSubnets: ic.Platform.AWS.VPC.Subnets, } - err := ValidateForProvisioning(route53Client, editedInstallConfig, meta) + err := ValidateForProvisioning(route53Client, ic, meta) if test.expectedErr == "" { assert.NoError(t, err) } else { @@ -1137,7 +1256,7 @@ func TestGetSubDomainDNSRecords(t *testing.T) { }, } - validDomainOutput := createBaseDomainHostedZone() + validDomainOutput := createBaseDomainHostedZoneOutput() mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -1146,10 +1265,7 @@ func TestGetSubDomainDNSRecords(t *testing.T) { for _, test := range cases { t.Run(test.name, func(t *testing.T) { - - ic := validInstallConfig() - ic.BaseDomain = test.baseDomain - + ic := icBuild.build(icBuild.withBaseDomain(test.baseDomain)) if test.expectedErr != "" { if test.problematicRecords == nil { route53Client.EXPECT().GetSubDomainDNSRecords(&validDomainOutput, ic, gomock.Any()).Return(nil, fmt.Errorf("%s", test.expectedErr)).AnyTimes() @@ -1208,8 +1324,7 @@ func TestSkipRecords(t *testing.T) { } // create the dottedClusterDomain in the same manner that it will be used in GetSubDomainDNSRecords - ic := validInstallConfig() - ic.BaseDomain = validDomainName + ic := icBuild.build() dottedClusterDomain := ic.ClusterDomain() + "." for _, test := range cases { @@ -1218,3 +1333,532 @@ func TestSkipRecords(t *testing.T) { }) } } + +func validAvailRegions() []string { + return []string{"us-east-1", "us-central-1"} +} + +func validAvailZones() []string { + return []string{"a", "b", "c"} +} + +func validEdgeAvailZones() []string { + return []string{"edge-a", "edge-b", "edge-c"} +} + +func validSubnets(subnetType string) Subnets { + switch subnetType { + case "edge": + return Subnets{ + "subnet-valid-public-edge-a": { + ID: "subnet-valid-public-edge-a", + Zone: &Zone{Name: "edge-a"}, + CIDR: "10.0.7.0/24", + Public: true, + }, + "subnet-valid-public-edge-b": { + ID: "subnet-valid-public-edge-b", + Zone: &Zone{Name: "edge-b"}, + CIDR: "10.0.8.0/24", + Public: true, + }, + "subnet-valid-public-edge-c": { + ID: "subnet-valid-public-edge-c", + Zone: &Zone{Name: "edge-c"}, + CIDR: "10.0.9.0/24", + Public: true, + }, + } + case "public": + return Subnets{ + "subnet-valid-public-a": { + ID: "subnet-valid-public-a", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.4.0/24", + Public: true, + }, + "subnet-valid-public-b": { + ID: "subnet-valid-public-b", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.5.0/24", + Public: true, + }, + "subnet-valid-public-c": { + ID: "subnet-valid-public-c", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.6.0/24", + Public: true, + }, + } + case "private": + return Subnets{ + "subnet-valid-private-a": { + ID: "subnet-valid-private-a", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.1.0/24", + Public: false, + }, + "subnet-valid-private-b": { + ID: "subnet-valid-private-b", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.2.0/24", + Public: false, + }, + "subnet-valid-private-c": { + ID: "subnet-valid-private-c", + Zone: &Zone{Name: "c"}, + CIDR: "10.0.3.0/24", + Public: false, + }, + } + } + return nil +} + +// byoSubnetsWithRoles returns a valid collection of subnets +// with assigned roles. +func byoSubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-private-a", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-private-b", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-private-c", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-a", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-b", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-c", + Roles: []aws.SubnetRole{ + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + {Type: aws.BootstrapNodeSubnetRole}, + }, + }, + } +} + +// byoPublicOnlySubnetsWithRoles returns a valid collection of subnets +// with assigned roles for a public-only cluster. +func byoPublicOnlySubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-public-a", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-b", + Roles: []aws.SubnetRole{ + {Type: aws.ClusterNodeSubnetRole}, + {Type: aws.ControlPlaneInternalLBSubnetRole}, + {Type: aws.ControlPlaneExternalLBSubnetRole}, + {Type: aws.IngressControllerLBSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-c", + Roles: []aws.SubnetRole{ + {Type: aws.BootstrapNodeSubnetRole}, + }, + }, + } +} + +// byoEdgeSubnetsWithRoles returns a valid collection of edge subnets +// with assigned EdgeNode roles. +func byoEdgeSubnetsWithRoles() []aws.Subnet { + return []aws.Subnet{ + { + ID: "subnet-valid-public-edge-a", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-edge-b", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + { + ID: "subnet-valid-public-edge-c", + Roles: []aws.SubnetRole{ + {Type: aws.EdgeNodeSubnetRole}, + }, + }, + } +} + +func otherTaggedPrivateSubnets() Subnets { + return Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.4.0/24", + Tags: Tags{ + TagNameKubernetesClusterPrefix + "other-cluster": "owned", + }, + }, + "subnet-valid-private-b1": { + ID: "subnet-valid-private-b1", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.5.0/24", + Tags: Tags{ + TagNameKubernetesUnmanaged: "true", + }, + }, + } +} + +func otherUntaggedPrivateSubnets() Subnets { + return Subnets{ + "subnet-valid-private-a1": { + ID: "subnet-valid-private-a1", + Zone: &Zone{Name: "a"}, + CIDR: "10.0.6.0/24", + }, + "subnet-valid-private-b1": { + ID: "subnet-valid-private-b1", + Zone: &Zone{Name: "b"}, + CIDR: "10.0.7.0/24", + }, + } +} + +func validServiceEndpoints() []aws.ServiceEndpoint { + return []aws.ServiceEndpoint{{ + Name: "ec2", + URL: "custom.ec2.us-east-1.amazonaws.com", + }, { + Name: "s3", + URL: "custom.s3.us-east-1.amazonaws.com", + }, { + Name: "iam", + URL: "custom.iam.us-east-1.amazonaws.com", + }, { + Name: "elasticloadbalancing", + URL: "custom.elasticloadbalancing.us-east-1.amazonaws.com", + }, { + Name: "tagging", + URL: "custom.tagging.us-east-1.amazonaws.com", + }, { + Name: "route53", + URL: "custom.route53.us-east-1.amazonaws.com", + }, { + Name: "sts", + URL: "custom.route53.us-east-1.amazonaws.com", + }} +} + +func invalidServiceEndpoint() []aws.ServiceEndpoint { + return []aws.ServiceEndpoint{{ + Name: "ec3", + URL: "bad-aws-endpoint", + }, { + Name: "route55", + URL: "http://bad-aws-endpoint.non", + }} +} + +func validInstanceTypes() map[string]InstanceType { + return map[string]InstanceType{ + "t2.small": { + DefaultVCpus: 1, + MemInMiB: 2048, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m5.large": { + DefaultVCpus: 2, + MemInMiB: 8192, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m5.xlarge": { + DefaultVCpus: 4, + MemInMiB: 16384, + Arches: []string{ec2.ArchitectureTypeX8664}, + }, + "m6g.xlarge": { + DefaultVCpus: 4, + MemInMiB: 16384, + Arches: []string{ec2.ArchitectureTypeArm64}, + }, + } +} + +func createBaseDomainHostedZoneOutput() route53.HostedZone { + return route53.HostedZone{ + CallerReference: &validCallerRef, + Id: &validDSId, + Name: &validDomainName, + } +} + +func createValidHostedZoneOutput() route53.GetHostedZoneOutput { + ptrValidNameServers := []*string{} + for i := range validNameServers { + ptrValidNameServers = append(ptrValidNameServers, &validNameServers[i]) + } + + validDelegationSet := route53.DelegationSet{CallerReference: &validCallerRef, Id: &validDSId, NameServers: ptrValidNameServers} + validHostedZone := route53.HostedZone{CallerReference: &validCallerRef, Id: &validDSId, Name: &validHostedZoneName} + validVPCs := []*route53.VPC{{VPCId: &validVPCID, VPCRegion: &validAvailRegions()[0]}} + + return route53.GetHostedZoneOutput{ + DelegationSet: &validDelegationSet, + HostedZone: &validHostedZone, + VPCs: validVPCs, + } +} + +type icOption func(*types.InstallConfig) +type icBuildForAWS struct{} + +var icBuild icBuildForAWS + +func (icBuild icBuildForAWS) build(opts ...icOption) *types.InstallConfig { + ic := &types.InstallConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: metaName, + }, + BaseDomain: validDomainName, + Publish: types.ExternalPublishingStrategy, + ControlPlane: &types.MachinePool{ + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }, + Compute: []types.MachinePool{{ + Name: types.MachinePoolComputeRoleName, + Architecture: types.ArchitectureAMD64, + Replicas: ptr.To[int64](3), + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }}, + Networking: &types.Networking{ + MachineNetwork: []types.MachineNetworkEntry{ + {CIDR: *ipnet.MustParseCIDR(validCIDR)}, + }, + }, + Platform: types.Platform{ + AWS: &aws.Platform{ + Region: validAvailRegions()[0], + }, + }, + } + for _, opt := range opts { + if opt != nil { + opt(ic) + } + } + return ic +} + +func (icBuild icBuildForAWS) withBaseBYO() icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Platform.AWS.Zones = validAvailZones() + ic.Compute[0].Platform.AWS.Zones = validAvailZones() + + subnetIDs := append(validSubnets("public").IDs(), validSubnets("private").IDs()...) + ic.AWS.VPC.Subnets = append(ic.AWS.VPC.Subnets, subnetsFromIDs(subnetIDs)...) + ic.AWS.HostedZone = validHostedZoneName + } +} + +func (icBuild icBuildForAWS) withPublish(publish types.PublishingStrategy) icOption { + return func(ic *types.InstallConfig) { + ic.Publish = publish + } +} + +func (icBuild icBuildForAWS) withHostedZone(hostedZone string) icOption { + return func(ic *types.InstallConfig) { + ic.AWS.HostedZone = hostedZone + } +} + +func (icBuild icBuildForAWS) withBaseDomain(baseDomain string) icOption { + return func(ic *types.InstallConfig) { + ic.BaseDomain = baseDomain + } +} + +func (icBuild icBuildForAWS) withVPCSubnets(subnets []aws.Subnet, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.AWS.VPC.Subnets = subnets + } else { + ic.AWS.VPC.Subnets = append(ic.AWS.VPC.Subnets, subnets...) + } + } +} + +func (icBuild icBuildForAWS) withVPCSubnetIDs(subnetIDs []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCSubnets(subnetsFromIDs(subnetIDs), overwrite)(ic) + } +} + +func (icBuild icBuildForAWS) withVPCEdgeSubnets(subnets []aws.Subnet, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCSubnets(subnets, overwrite)(ic) + if len(subnets) > 0 { + icBuild.withComputeMachinePool([]types.MachinePool{{ + Name: types.MachinePoolEdgeRoleName, + Platform: types.MachinePoolPlatform{ + AWS: &aws.MachinePool{}, + }, + }}, false)(ic) + } + } +} + +func (icBuild icBuildForAWS) withVPCEdgeSubnetIDs(subnetIDs []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + icBuild.withVPCEdgeSubnets(subnetsFromIDs(subnetIDs), overwrite)(ic) + } +} + +func (icBuild icBuildForAWS) withControlPlaneMachinePool(machinePool types.MachinePool) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane = &machinePool + } +} + +func (icBuild icBuildForAWS) withComputeMachinePool(machinePools []types.MachinePool, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Compute = machinePools + } else { + ic.Compute = append(ic.Compute, machinePools...) + } + } +} + +func (icBuild icBuildForAWS) withControlPlanePlatformZones(zones []string, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.ControlPlane.Platform.AWS.Zones = zones + } else { + ic.ControlPlane.Platform.AWS.Zones = append(ic.ControlPlane.Platform.AWS.Zones, zones...) + } + } +} + +func (icBuild icBuildForAWS) withComputePlatformZones(zones []string, overwrite bool, index int) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Compute[index].Platform.AWS.Zones = zones + } else { + ic.Compute[index].Platform.AWS.Zones = append(ic.Compute[index].Platform.AWS.Zones, zones...) + } + } +} + +func (icBuild icBuildForAWS) withControlPlanePlatformAMI(amiID string) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withComputePlatformAMI(amiID string, index int) icOption { + return func(ic *types.InstallConfig) { + ic.Compute[index].Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withComputeReplicas(replicas int64, index int) icOption { + return func(ic *types.InstallConfig) { + ic.Compute[index].Replicas = ptr.To(replicas) + } +} + +func (icBuild icBuildForAWS) withInstanceType(defaultInstanceType string, ctrPlaneInstanceType string, computeInstanceTypes ...string) icOption { + return func(ic *types.InstallConfig) { + if ic.Platform.AWS.DefaultMachinePlatform == nil { + icBuild.withDefaultPlatformMachine(aws.MachinePool{})(ic) + } + ic.Platform.AWS.DefaultMachinePlatform.InstanceType = defaultInstanceType + ic.ControlPlane.Platform.AWS.InstanceType = ctrPlaneInstanceType + for idx, instanceType := range computeInstanceTypes { + ic.Compute[idx].Platform.AWS.InstanceType = instanceType + } + } +} + +func (icBuild icBuildForAWS) withInstanceArchitecture(ctrPlaneInstanceArch types.Architecture, computeInstanceArchs ...types.Architecture) icOption { + return func(ic *types.InstallConfig) { + ic.ControlPlane.Architecture = ctrPlaneInstanceArch + for idx, arch := range computeInstanceArchs { + ic.Compute[idx].Architecture = arch + } + } +} + +func (icBuild icBuildForAWS) withPlatformRegion(region string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.Region = region + } +} + +func (icBuild icBuildForAWS) withPlatformAMIID(amiID string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.AMIID = amiID + } +} + +func (icBuild icBuildForAWS) withServiceEndpoints(endpoints []aws.ServiceEndpoint, overwrite bool) icOption { + return func(ic *types.InstallConfig) { + if overwrite { + ic.Platform.AWS.ServiceEndpoints = endpoints + } else { + ic.Platform.AWS.ServiceEndpoints = append(ic.Platform.AWS.ServiceEndpoints, endpoints...) + } + } +} + +func (icBuild icBuildForAWS) withDefaultPlatformMachine(awsMachine aws.MachinePool) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.DefaultMachinePlatform = &awsMachine + } +} + +func (icBuild icBuildForAWS) withPublicIPv4Pool(publicIPv4Pool string) icOption { + return func(ic *types.InstallConfig) { + ic.Platform.AWS.PublicIpv4Pool = publicIPv4Pool + } +}