1
0
mirror of https://github.com/openshift/installer.git synced 2026-02-05 06:46:36 +01:00

Merge pull request #9599 from tthvo/CORS-3870

CORS-3870: add validations for subnets field with AWS API
This commit is contained in:
openshift-merge-bot[bot]
2025-04-15 00:50:38 +00:00
committed by GitHub
6 changed files with 2288 additions and 1188 deletions

View File

@@ -26,15 +26,14 @@ type Metadata struct {
availabilityZones []string
availableRegions []string
edgeZones []string
privateSubnets Subnets
publicSubnets Subnets
edgeSubnets Subnets
subnets SubnetGroups
vpcSubnets SubnetGroups
vpc string
instanceTypes map[string]InstanceType
Region string `json:"region,omitempty"`
Subnets []typesaws.Subnet `json:"subnets,omitempty"`
Services []typesaws.ServiceEndpoint `json:"services,omitempty"`
Region string `json:"region,omitempty"`
ProvidedSubnets []typesaws.Subnet `json:"subnets,omitempty"`
Services []typesaws.ServiceEndpoint `json:"services,omitempty"`
ec2Client *ec2.Client
@@ -43,7 +42,7 @@ type Metadata struct {
// NewMetadata initializes a new Metadata object.
func NewMetadata(region string, subnets []typesaws.Subnet, services []typesaws.ServiceEndpoint) *Metadata {
return &Metadata{Region: region, Subnets: subnets, Services: services}
return &Metadata{Region: region, ProvidedSubnets: subnets, Services: services}
}
// Session holds an AWS session which can be used for AWS API calls
@@ -174,7 +173,7 @@ func (m *Metadata) EdgeSubnets(ctx context.Context) (Subnets, error) {
if err != nil {
return nil, fmt.Errorf("error retrieving Edge Subnets: %w", err)
}
return m.edgeSubnets, nil
return m.subnets.Edge, nil
}
// SetZoneAttributes retrieves AWS Zone attributes and update required fields in zones.
@@ -239,7 +238,7 @@ func (m *Metadata) PrivateSubnets(ctx context.Context) (Subnets, error) {
if err != nil {
return nil, fmt.Errorf("error retrieving Private Subnets: %w", err)
}
return m.privateSubnets, nil
return m.subnets.Private, nil
}
// PublicSubnets retrieves subnet metadata indexed by subnet ID, for
@@ -250,7 +249,29 @@ func (m *Metadata) PublicSubnets(ctx context.Context) (Subnets, error) {
if err != nil {
return nil, fmt.Errorf("error retrieving Public Subnets: %w", err)
}
return m.publicSubnets, nil
return m.subnets.Public, nil
}
// Subnets retrieves a group of subnet metadata that is indexed by subnet ID for all provided subnets.
// This includes private, public and edge subnets.
func (m *Metadata) Subnets(ctx context.Context) (SubnetGroups, error) {
err := m.populateSubnets(ctx)
if err != nil {
return m.subnets, fmt.Errorf("error retrieving all Subnets: %w", err)
}
return m.subnets, nil
}
// VPCSubnets retrieves a group of all subnet metadata that is indexed by subnet ID in the VPC of the provided subnets.
// These include cluster subnets (i.e. provided in the installconfig) and potentially other non-cluster subnets in the VPC.
//
// This func is only used for validations. Use func Subnets to select only cluster subnets.
func (m *Metadata) VPCSubnets(ctx context.Context) (SubnetGroups, error) {
err := m.populateVPCSubnets(ctx)
if err != nil {
return m.vpcSubnets, fmt.Errorf("error retrieving Subnets in VPC: %w", err)
}
return m.vpcSubnets, nil
}
// VPC retrieves the VPC ID containing PublicSubnets and PrivateSubnets.
@@ -269,49 +290,75 @@ func (m *Metadata) SubnetByID(ctx context.Context, subnetID string) (subnet Subn
return subnet, fmt.Errorf("error retrieving subnet for ID %s: %w", subnetID, err)
}
if subnet, ok := m.privateSubnets[subnetID]; ok {
if subnet, ok := m.subnets.Private[subnetID]; ok {
return subnet, nil
}
if subnet, ok := m.publicSubnets[subnetID]; ok {
if subnet, ok := m.subnets.Public[subnetID]; ok {
return subnet, nil
}
if subnet, ok := m.edgeSubnets[subnetID]; ok {
if subnet, ok := m.subnets.Edge[subnetID]; ok {
return subnet, nil
}
return subnet, fmt.Errorf("no subnet found for ID %s", subnetID)
}
// populateSubnets retrieves metadata for provided subnets.
func (m *Metadata) populateSubnets(ctx context.Context) error {
m.mutex.Lock()
defer m.mutex.Unlock()
if len(m.Subnets) == 0 {
if len(m.ProvidedSubnets) == 0 {
return errors.New("no subnets configured")
}
if m.vpc != "" || len(m.privateSubnets) > 0 || len(m.publicSubnets) > 0 || len(m.edgeSubnets) > 0 {
subnetGroups := m.subnets
if m.vpc != "" || len(subnetGroups.Private) > 0 || len(subnetGroups.Public) > 0 || len(subnetGroups.Edge) > 0 {
// Call to populate subnets has already happened
return nil
}
session, err := m.unlockedSession(ctx)
client, err := m.EC2Client(ctx)
if err != nil {
return err
}
subnetIDs := make([]string, len(m.Subnets))
for i, subnet := range m.Subnets {
subnetIDs := make([]string, len(m.ProvidedSubnets))
for i, subnet := range m.ProvidedSubnets {
subnetIDs[i] = string(subnet.ID)
}
sb, err := subnets(ctx, session, m.Region, subnetIDs)
sb, err := subnets(ctx, client, subnetIDs, "")
m.vpc = sb.VPC
m.privateSubnets = sb.Private
m.publicSubnets = sb.Public
m.edgeSubnets = sb.Edge
m.subnets = sb
return err
}
// populateVPCSubnets retrieves metadata for all subnets in the VPC of provided subnets.
func (m *Metadata) populateVPCSubnets(ctx context.Context) error {
// we need to populate provided subnets to get the VPC ID.
if err := m.populateSubnets(ctx); err != nil {
return err
}
m.mutex.Lock()
defer m.mutex.Unlock()
vpcSubnetGroups := m.vpcSubnets
if len(vpcSubnetGroups.Private) > 0 || len(vpcSubnetGroups.Public) > 0 || len(vpcSubnetGroups.Edge) > 0 {
// Call to populate subnets has already happened
return nil
}
client, err := m.EC2Client(ctx)
if err != nil {
return err
}
sb, err := subnets(ctx, client, nil, m.vpc)
m.vpcSubnets = sb
return err
}

View File

@@ -5,7 +5,9 @@ import (
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"github.com/openshift/installer/pkg/ipnet"
"github.com/openshift/installer/pkg/types"
"github.com/openshift/installer/pkg/types/aws"
)
@@ -21,6 +23,58 @@ func basicInstallConfig() types.InstallConfig {
}
}
// validBYOSubnetsInstallConfig returns a valid install config for BYO subnets use case.
// Test cases can unset fields if necessary.
func validBYOSubnetsInstallConfig() *types.InstallConfig {
return &types.InstallConfig{
Networking: &types.Networking{
MachineNetwork: []types.MachineNetworkEntry{
{CIDR: *ipnet.MustParseCIDR(validCIDR)},
},
},
BaseDomain: validDomainName,
Publish: types.ExternalPublishingStrategy,
Platform: types.Platform{
AWS: &aws.Platform{
Region: "us-east-1",
VPC: aws.VPC{
Subnets: []aws.Subnet{
{ID: "subnet-valid-private-a"},
{ID: "subnet-valid-private-b"},
{ID: "subnet-valid-private-c"},
{ID: "subnet-valid-public-a"},
{ID: "subnet-valid-public-b"},
{ID: "subnet-valid-public-c"},
},
},
HostedZone: validHostedZoneName,
},
},
ControlPlane: &types.MachinePool{
Architecture: types.ArchitectureAMD64,
Replicas: ptr.To[int64](3),
Platform: types.MachinePoolPlatform{
AWS: &aws.MachinePool{
Zones: []string{"a", "b", "c"},
},
},
},
Compute: []types.MachinePool{{
Name: types.MachinePoolComputeRoleName,
Architecture: types.ArchitectureAMD64,
Replicas: ptr.To[int64](3),
Platform: types.MachinePoolPlatform{
AWS: &aws.MachinePool{
Zones: []string{"a", "b", "c"},
},
},
}},
ObjectMeta: metav1.ObjectMeta{
Name: metaName,
},
}
}
func TestIncludesCreateInstanceRole(t *testing.T) {
t.Run("Should be true when", func(t *testing.T) {
t.Run("no machine types specified", func(t *testing.T) {
@@ -310,28 +364,28 @@ func TestIAMRolePermissions(t *testing.T) {
t.Run("Should include", func(t *testing.T) {
t.Run("create and delete shared IAM role permissions", func(t *testing.T) {
t.Run("when role specified for controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceRole)
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when instance profile specified for controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceRole)
assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when role specified for compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceRole)
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when instance profile specified for compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceRole)
@@ -340,7 +394,7 @@ func TestIAMRolePermissions(t *testing.T) {
})
t.Run("create IAM role permissions", func(t *testing.T) {
t.Run("when no existing roles and instance profiles are specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceRole)
assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
@@ -350,7 +404,7 @@ func TestIAMRolePermissions(t *testing.T) {
t.Run("Should not include create IAM role permissions", func(t *testing.T) {
t.Run("when role specified for defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{
IAMRole: "custom-default-role",
}
@@ -359,7 +413,7 @@ func TestIAMRolePermissions(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when role specified for controlPlane and compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMRole = "custom-master-role"
ic.Compute[0].Platform.AWS.IAMRole = "custom-worker-role"
requiredPerms := RequiredPermissionGroups(ic)
@@ -367,7 +421,7 @@ func TestIAMRolePermissions(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{
IAMProfile: "custom-default-profile",
}
@@ -376,7 +430,7 @@ func TestIAMRolePermissions(t *testing.T) {
assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceRole)
})
t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile"
ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile"
requiredPerms := RequiredPermissionGroups(ic)
@@ -390,14 +444,14 @@ func TestIAMProfilePermissions(t *testing.T) {
t.Run("Should include", func(t *testing.T) {
t.Run("create and delete shared instance profile permissions", func(t *testing.T) {
t.Run("when instance profile specified for controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile)
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile)
})
t.Run("when instance profile specified for compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile)
@@ -406,7 +460,7 @@ func TestIAMProfilePermissions(t *testing.T) {
})
t.Run("create instance profile permissions", func(t *testing.T) {
t.Run("when no existing instance profiles are specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateInstanceProfile)
assert.NotContains(t, requiredPerms, PermissionDeleteSharedInstanceProfile)
@@ -416,7 +470,7 @@ func TestIAMProfilePermissions(t *testing.T) {
t.Run("Should not include create instance profile permissions", func(t *testing.T) {
t.Run("when instance profile specified for defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{
IAMProfile: "custom-default-profile",
}
@@ -425,7 +479,7 @@ func TestIAMProfilePermissions(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionDeleteSharedInstanceProfile)
})
t.Run("when instance profile specified for controlPlane and compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.IAMProfile = "custom-master-profile"
ic.Compute[0].Platform.AWS.IAMProfile = "custom-worker-profile"
requiredPerms := RequiredPermissionGroups(ic)
@@ -527,7 +581,7 @@ func TestIncludesKMSEncryptionKeys(t *testing.T) {
func TestKMSKeyPermissions(t *testing.T) {
t.Run("Should include KMS key permissions", func(t *testing.T) {
t.Run("when KMS key specified for controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.EC2RootVolume = aws.EC2RootVolume{
KMSKeyARN: "custom-master-key",
}
@@ -535,7 +589,7 @@ func TestKMSKeyPermissions(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys)
})
t.Run("when KMS key specified for compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.EC2RootVolume = aws.EC2RootVolume{
KMSKeyARN: "custom-worker-key",
}
@@ -543,7 +597,7 @@ func TestKMSKeyPermissions(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionKMSEncryptionKeys)
})
t.Run("when KMS key specified for defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{
EC2RootVolume: aws.EC2RootVolume{
KMSKeyARN: "custom-default-key",
@@ -556,14 +610,14 @@ func TestKMSKeyPermissions(t *testing.T) {
t.Run("Should not include KMS key permissions", func(t *testing.T) {
t.Run("when no machine types specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane = nil
ic.Compute = nil
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys)
})
t.Run("when no KMS keys specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{}
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionKMSEncryptionKeys)
@@ -575,14 +629,14 @@ func TestVPCPermissions(t *testing.T) {
t.Run("Should include", func(t *testing.T) {
t.Run("create network permissions when VPC not specified", func(t *testing.T) {
t.Run("for standard regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.VPC.Subnets = nil
ic.AWS.HostedZone = ""
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateNetworking)
})
t.Run("for secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.Region = "us-iso-east-1"
ic.AWS.VPC.Subnets = nil
ic.AWS.HostedZone = ""
@@ -591,32 +645,32 @@ func TestVPCPermissions(t *testing.T) {
})
})
t.Run("delete network permissions when VPC not specified for standard region", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.VPC.Subnets = nil
ic.AWS.HostedZone = ""
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionDeleteNetworking)
})
t.Run("delete shared network permissions when VPC specified for standard region", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionDeleteSharedNetworking)
})
})
t.Run("Should not include", func(t *testing.T) {
t.Run("create network permissions when VPC specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionCreateNetworking)
})
t.Run("delete network permissions", func(t *testing.T) {
t.Run("when VPC specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteNetworking)
})
t.Run("on secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.Region = "us-iso-east-1"
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteNetworking)
@@ -624,14 +678,14 @@ func TestVPCPermissions(t *testing.T) {
})
t.Run("delete shared network permissions", func(t *testing.T) {
t.Run("when VPC not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.VPC.Subnets = nil
ic.AWS.HostedZone = ""
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking)
})
t.Run("on secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.Region = "us-iso-east-1"
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteSharedNetworking)
@@ -643,13 +697,13 @@ func TestVPCPermissions(t *testing.T) {
func TestPrivateZonePermissions(t *testing.T) {
t.Run("Should include", func(t *testing.T) {
t.Run("create hosted zone permissions when PHZ not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.HostedZone = ""
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateHostedZone)
})
t.Run("delete hosted zone permissions when PHZ not specified on standard regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.HostedZone = ""
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionDeleteHostedZone)
@@ -657,18 +711,18 @@ func TestPrivateZonePermissions(t *testing.T) {
})
t.Run("Should not include", func(t *testing.T) {
t.Run("create hosted zone permissions when PHZ specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionCreateHostedZone)
})
t.Run("delete hosted zone permissions", func(t *testing.T) {
t.Run("on secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone)
})
t.Run("when PHZ specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteHostedZone)
})
@@ -678,13 +732,13 @@ func TestPrivateZonePermissions(t *testing.T) {
func TestPublicIPv4PoolPermissions(t *testing.T) {
t.Run("Should include IPv4Pool permissions when IPv4 pool specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.PublicIpv4Pool = "custom-ipv4-pool"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionPublicIpv4Pool)
})
t.Run("Should not include IPv4Pool permissions when IPv4 pool not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionPublicIpv4Pool)
})
@@ -694,25 +748,25 @@ func TestBasePermissions(t *testing.T) {
t.Run("Should include", func(t *testing.T) {
t.Run("base create permissions", func(t *testing.T) {
t.Run("on standard regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateBase)
})
t.Run("on secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.Region = "us-iso-east-1"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionCreateBase)
})
})
t.Run("base delete permissions on standard regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionDeleteBase)
})
})
t.Run("Should not include base delete permissions on secret regions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.Region = "us-iso-east-1"
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteBase)
@@ -721,12 +775,12 @@ func TestBasePermissions(t *testing.T) {
func TestDeleteIgnitionPermissions(t *testing.T) {
t.Run("Should include delete ignition permissions", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionDeleteIgnitionObjects)
})
t.Run("Should not include delete ignition permission when specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.BestEffortDeleteIgnition = true
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDeleteIgnitionObjects)
@@ -737,7 +791,7 @@ func TestIncludesInstanceType(t *testing.T) {
const instanceType = "m7a.2xlarge"
t.Run("Should be true when instance type specified for", func(t *testing.T) {
t.Run("defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.DefaultMachinePlatform = &aws.MachinePool{
InstanceType: instanceType,
}
@@ -745,20 +799,20 @@ func TestIncludesInstanceType(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionValidateInstanceType)
})
t.Run("controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.InstanceType = instanceType
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionValidateInstanceType)
})
t.Run("compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.InstanceType = instanceType
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionValidateInstanceType)
})
})
t.Run("Should be false when instance type is not set", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
assert.NotContains(t, RequiredPermissionGroups(ic), PermissionValidateInstanceType)
})
}
@@ -766,7 +820,7 @@ func TestIncludesInstanceType(t *testing.T) {
func TestIncludesZones(t *testing.T) {
t.Run("Should be true when", func(t *testing.T) {
t.Run("zones specified in defaultMachinePlatform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.Zones = []string{}
ic.Compute[0].Platform.AWS.Zones = []string{}
ic.AWS.VPC.Subnets = []aws.Subnet{}
@@ -777,21 +831,21 @@ func TestIncludesZones(t *testing.T) {
assert.NotContains(t, requiredPerms, PermissionDefaultZones)
})
t.Run("zones specified in controlPlane", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute[0].Platform.AWS.Zones = []string{}
ic.AWS.VPC.Subnets = []aws.Subnet{}
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDefaultZones)
})
t.Run("zones specified in compute", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.Zones = []string{}
ic.AWS.VPC.Subnets = []aws.Subnet{}
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionDefaultZones)
})
t.Run("subnets specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.ControlPlane.Platform.AWS.Zones = []string{}
ic.Compute[0].Platform.AWS.Zones = []string{}
requiredPerms := RequiredPermissionGroups(ic)
@@ -799,7 +853,7 @@ func TestIncludesZones(t *testing.T) {
})
})
t.Run("Should be false when neither zones nor subnets specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.VPC.Subnets = []aws.Subnet{}
ic.ControlPlane.Platform.AWS.Zones = []string{}
ic.Compute[0].Platform.AWS.Zones = []string{}
@@ -810,13 +864,13 @@ func TestIncludesZones(t *testing.T) {
func TestIncludesAssumeRole(t *testing.T) {
t.Run("Should be true when IAM role specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.AWS.HostedZoneRole = "custom-role"
requiredPerms := RequiredPermissionGroups(ic)
assert.Contains(t, requiredPerms, PermissionAssumeRole)
})
t.Run("Should be false when IAM role not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionAssumeRole)
})
@@ -824,7 +878,7 @@ func TestIncludesAssumeRole(t *testing.T) {
func TestIncludesWavelengthZones(t *testing.T) {
t.Run("Should be true when edge compute specified with WL zones", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute = append(ic.Compute, types.MachinePool{
Name: "edge",
Platform: types.MachinePoolPlatform{
@@ -838,7 +892,7 @@ func TestIncludesWavelengthZones(t *testing.T) {
})
t.Run("Should be false when", func(t *testing.T) {
t.Run("edge compute specified without WL zones", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute = append(ic.Compute, types.MachinePool{
Name: "edge",
Platform: types.MachinePoolPlatform{
@@ -851,7 +905,7 @@ func TestIncludesWavelengthZones(t *testing.T) {
assert.NotContains(t, requiredPerms, PermissionCarrierGateway)
})
t.Run("edge compute not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionCarrierGateway)
})
@@ -861,7 +915,7 @@ func TestIncludesWavelengthZones(t *testing.T) {
func TestIncludesEdgeDefaultInstance(t *testing.T) {
t.Run("Should be true when at least one edge compute pool specified", func(t *testing.T) {
t.Run("without platform", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute = append(ic.Compute, types.MachinePool{
Name: "edge",
})
@@ -878,7 +932,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) {
assert.Contains(t, requiredPerms, PermissionEdgeDefaultInstance)
})
t.Run("without instance type", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute = append(ic.Compute, types.MachinePool{
Name: "edge",
Platform: types.MachinePoolPlatform{
@@ -902,7 +956,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) {
})
t.Run("Should be false when", func(t *testing.T) {
t.Run("edge compute specified with instance type", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
ic.Compute = append(ic.Compute, types.MachinePool{
Name: "edge",
Platform: types.MachinePoolPlatform{
@@ -916,7 +970,7 @@ func TestIncludesEdgeDefaultInstance(t *testing.T) {
assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance)
})
t.Run("edge compute not specified", func(t *testing.T) {
ic := validInstallConfig()
ic := validBYOSubnetsInstallConfig()
requiredPerms := RequiredPermissionGroups(ic)
assert.NotContains(t, requiredPerms, PermissionEdgeDefaultInstance)
})

View File

@@ -3,12 +3,13 @@ package aws
import (
"context"
"fmt"
"maps"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/sirupsen/logrus"
"k8s.io/utils/ptr"
typesaws "github.com/openshift/installer/pkg/types/aws"
)
@@ -29,11 +30,23 @@ type Subnet struct {
// Public is the flag to define the subnet public.
Public bool
// Tags is the map of the subnet's tags.
Tags Tags
}
// Subnets is the map for the Subnet metadata indexed by subnetID.
type Subnets map[string]Subnet
// IDs returns the subnet IDs (i.e. map keys) in the Subnets.
func (sns Subnets) IDs() []string {
subnetIDs := make([]string, 0)
for id := range sns {
subnetIDs = append(subnetIDs, id)
}
return subnetIDs
}
// SubnetsByZone is the map for the Subnet metadata indexed by zone.
type SubnetsByZone map[string]Subnet
@@ -45,92 +58,82 @@ type SubnetGroups struct {
VPC string
}
// subnets retrieves metadata for the given subnet(s).
func subnets(ctx context.Context, session *session.Session, region string, ids []string) (subnetGroups SubnetGroups, err error) {
metas := make(Subnets, len(ids))
zoneNames := make([]*string, len(ids))
availabilityZones := make(map[string]*ec2.AvailabilityZone, len(ids))
subnetGroups = SubnetGroups{
Public: make(Subnets, len(ids)),
Private: make(Subnets, len(ids)),
Edge: make(Subnets, len(ids)),
// subnets retrieves metadata for the given subnet(s) or VPC.
func subnets(ctx context.Context, client *ec2.Client, subnetIDs []string, vpcID string) (SubnetGroups, error) {
metas := make(Subnets, len(subnetIDs))
zoneNames := make([]string, 0)
availabilityZones := make(map[string]ec2types.AvailabilityZone, len(subnetIDs))
subnetGroups := SubnetGroups{
Public: make(Subnets, len(subnetIDs)),
Private: make(Subnets, len(subnetIDs)),
Edge: make(Subnets, len(subnetIDs)),
}
var vpcFromSubnet string
client := ec2.New(session, aws.NewConfig().WithRegion(region))
idPointers := make([]*string, 0, len(ids))
for _, id := range ids {
idPointers = append(idPointers, aws.String(id))
subnetInput := &ec2.DescribeSubnetsInput{}
if len(vpcID) > 0 {
subnetInput.Filters = append(subnetInput.Filters, ec2types.Filter{
Name: ptr.To("vpc-id"),
Values: []string{vpcID},
})
}
if len(subnetIDs) > 0 {
subnetInput.SubnetIds = subnetIDs
}
var lastError error
err = client.DescribeSubnetsPagesWithContext(
ctx,
&ec2.DescribeSubnetsInput{SubnetIds: idPointers},
func(results *ec2.DescribeSubnetsOutput, lastPage bool) bool {
for _, subnet := range results.Subnets {
if subnet.SubnetId == nil {
continue
}
if subnet.SubnetArn == nil {
lastError = fmt.Errorf("%s has no ARN", *subnet.SubnetId)
return false
}
if subnet.VpcId == nil {
lastError = fmt.Errorf("%s has no VPC", *subnet.SubnetId)
return false
}
if subnet.AvailabilityZone == nil {
lastError = fmt.Errorf("%s has not availability zone", *subnet.SubnetId)
return false
}
if subnetGroups.VPC == "" {
subnetGroups.VPC = *subnet.VpcId
vpcFromSubnet = *subnet.SubnetId
} else if *subnet.VpcId != subnetGroups.VPC {
lastError = fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC)
return false
}
metas[aws.StringValue(subnet.SubnetId)] = Subnet{
ID: aws.StringValue(subnet.SubnetId),
ARN: aws.StringValue(subnet.SubnetArn),
Zone: &Zone{Name: aws.StringValue(subnet.AvailabilityZone)},
CIDR: aws.StringValue(subnet.CidrBlock),
Public: false,
}
zoneNames = append(zoneNames, subnet.AvailabilityZone)
err := describeSubnets(ctx, client, subnetInput, func(subnets []ec2types.Subnet) error {
var vpcFromSubnet string
for _, subnet := range subnets {
if subnet.SubnetId == nil {
continue
}
return !lastPage
},
)
if err == nil {
err = lastError
}
if len(ptr.Deref(subnet.SubnetArn, "")) == 0 {
return fmt.Errorf("%s has no ARN", *subnet.SubnetId)
}
if len(ptr.Deref(subnet.VpcId, "")) == 0 {
return fmt.Errorf("%s has no VPC", *subnet.SubnetId)
}
if len(ptr.Deref(subnet.AvailabilityZone, "")) == 0 {
return fmt.Errorf("%s has no availability zone", *subnet.SubnetId)
}
if subnetGroups.VPC == "" {
subnetGroups.VPC = *subnet.VpcId
vpcFromSubnet = *subnet.SubnetId
} else if *subnet.VpcId != subnetGroups.VPC {
return fmt.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC)
}
// At this point, we should be safe to dereference these fields.
metas[*subnet.SubnetId] = Subnet{
ID: *subnet.SubnetId,
ARN: *subnet.SubnetArn,
Zone: &Zone{Name: *subnet.AvailabilityZone},
CIDR: ptr.Deref(subnet.CidrBlock, ""),
Public: false,
Tags: FromAWSTags(subnet.Tags),
}
zoneNames = append(zoneNames, *subnet.AvailabilityZone)
}
return nil
})
if err != nil {
return subnetGroups, fmt.Errorf("describing subnets: %w", err)
return subnetGroups, err
}
var routeTables []*ec2.RouteTable
err = client.DescribeRouteTablesPagesWithContext(
ctx,
&ec2.DescribeRouteTablesInput{
Filters: []*ec2.Filter{{
Name: aws.String("vpc-id"),
Values: []*string{aws.String(subnetGroups.VPC)},
}},
},
func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool {
routeTables = append(routeTables, results.RouteTables...)
return !lastPage
},
)
var routeTables []ec2types.RouteTable
err = describeRouteTables(ctx, client, &ec2.DescribeRouteTablesInput{
Filters: []ec2types.Filter{{
Name: ptr.To("vpc-id"),
Values: []string{subnetGroups.VPC},
}},
}, func(rTables []ec2types.RouteTable) error {
routeTables = append(routeTables, rTables...)
return nil
})
if err != nil {
return subnetGroups, fmt.Errorf("describing route tables: %w", err)
return subnetGroups, err
}
azs, err := client.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames})
azs, err := client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames})
if err != nil {
return subnetGroups, fmt.Errorf("describing availability zones: %w", err)
}
@@ -140,6 +143,14 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
publicOnlySubnets := typesaws.IsPublicOnlySubnetsEnabled()
var ids []string
if len(vpcID) > 0 {
ids = metas.IDs()
}
if len(subnetIDs) > 0 {
ids = subnetIDs
}
for _, id := range ids {
meta, ok := metas[id]
if !ok {
@@ -157,10 +168,10 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
return subnetGroups, fmt.Errorf("unable to read properties of zone name %s from the list %v: %w", zoneName, zoneNames, err)
}
zone := availabilityZones[zoneName]
meta.Zone.Type = aws.StringValue(zone.ZoneType)
meta.Zone.GroupName = aws.StringValue(zone.GroupName)
meta.Zone.Type = ptr.Deref(zone.ZoneType, "")
meta.Zone.GroupName = ptr.Deref(zone.GroupName, "")
if availabilityZones[zoneName].ParentZoneName != nil {
meta.Zone.ParentZoneName = aws.StringValue(zone.ParentZoneName)
meta.Zone.ParentZoneName = ptr.Deref(zone.ParentZoneName, "")
}
// AWS Local Zones are grouped as Edge subnets
@@ -189,12 +200,12 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
}
// https://github.com/kubernetes/kubernetes/blob/9f036cd43d35a9c41d7ac4ca82398a6d0bef957b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L3376-L3419
func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
var subnetTable *ec2.RouteTable
func isSubnetPublic(rt []ec2types.RouteTable, subnetID string) (bool, error) {
var subnetTable *ec2types.RouteTable
for _, table := range rt {
for _, assoc := range table.Associations {
if aws.StringValue(assoc.SubnetId) == subnetID {
subnetTable = table
if ptr.Equal(assoc.SubnetId, &subnetID) {
subnetTable = &table
break
}
}
@@ -205,10 +216,10 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
// associated with the VPC's main routing table.
for _, table := range rt {
for _, assoc := range table.Associations {
if aws.BoolValue(assoc.Main) {
if ptr.Deref(assoc.Main, false) {
logrus.Debugf("Assuming implicit use of main routing table %s for %s",
aws.StringValue(table.RouteTableId), subnetID)
subnetTable = table
ptr.Deref(table.RouteTableId, ""), subnetID)
subnetTable = &table
break
}
}
@@ -226,13 +237,56 @@ func isSubnetPublic(rt []*ec2.RouteTable, subnetID string) (bool, error) {
// from the default in-subnet route which is called "local"
// or other virtual gateway (starting with vgv)
// or vpc peering connections (starting with pcx).
if strings.HasPrefix(aws.StringValue(route.GatewayId), "igw") {
if strings.HasPrefix(ptr.Deref(route.GatewayId, ""), "igw") {
return true, nil
}
if strings.HasPrefix(aws.StringValue(route.CarrierGatewayId), "cagw") {
if strings.HasPrefix(ptr.Deref(route.CarrierGatewayId, ""), "cagw") {
return true, nil
}
}
return false, nil
}
// describeSubnets retrieves metadata for subnets with given filters.
func describeSubnets(ctx context.Context, client *ec2.Client, input *ec2.DescribeSubnetsInput, fn func(subnets []ec2types.Subnet) error) error {
paginator := ec2.NewDescribeSubnetsPaginator(client, input)
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
return fmt.Errorf("describing subnets: %w", err)
}
// If the handler returns an error, we stop early to avoid extra API calls.
if err = fn(page.Subnets); err != nil {
return err
}
}
return nil
}
// describeRouteTables retrieves metadata for route tables with given filters.
func describeRouteTables(ctx context.Context, client *ec2.Client, input *ec2.DescribeRouteTablesInput, fn func(subnets []ec2types.RouteTable) error) error {
paginator := ec2.NewDescribeRouteTablesPaginator(client, input)
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
return fmt.Errorf("describing route tables: %w", err)
}
// If the handler returns an error, we stop early to avoid extra API calls.
if err = fn(page.RouteTables); err != nil {
return err
}
}
return nil
}
// mergeSubnets merged two or more Subnets into a single one for convenience.
func mergeSubnets(groups ...Subnets) Subnets {
subnets := make(Subnets)
for _, group := range groups {
maps.Copy(subnets, group)
}
return subnets
}

View File

@@ -0,0 +1,44 @@
package aws
import (
"strings"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
"k8s.io/utils/ptr"
)
const (
// TagNameKubernetesClusterPrefix is the tag name prefix used by CCM
// to differentiate multiple logically independent clusters running in the same AZ.
TagNameKubernetesClusterPrefix = "kubernetes.io/cluster/"
// TagNameKubernetesUnmanaged is the tag name to indicate that a resource is unmanaged
// by the cluster and should be ignored by CCM. For example, kubernetes.io/cluster/unmanaged=true.
TagNameKubernetesUnmanaged = TagNameKubernetesClusterPrefix + "unmanaged"
)
// Tags represents AWS resource tags as a map.
// This helps avoid iterating over the tag list for every lookup.
type Tags map[string]string
// FromAWSTags converts a list of AWS tags into a map.
func FromAWSTags(awsTags []types.Tag) Tags {
tags := make(Tags, len(awsTags))
for _, tag := range awsTags {
key, value := ptr.Deref(tag.Key, ""), ptr.Deref(tag.Value, "")
if len(key) > 0 {
tags[key] = value
}
}
return tags
}
// HasTagKeyPrefix returns true if there is a tag with a given key prefix.
func (t Tags) HasTagKeyPrefix(prefix string) bool {
for key := range t {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}

View File

@@ -50,7 +50,7 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig)
allErrs = append(allErrs, validateAMI(ctx, meta, config)...)
allErrs = append(allErrs, validatePublicIpv4Pool(ctx, meta, field.NewPath("platform", "aws", "publicIpv4PoolId"), config)...)
allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config.Platform.AWS, config.Networking, config.Publish)...)
allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config)...)
if awstypes.IsPublicOnlySubnetsEnabled() {
logrus.Warnln("Public-only subnets install. Please be warned this is not supported")
@@ -98,8 +98,9 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig)
return allErrs.ToAggregate()
}
func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList {
func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList {
allErrs := field.ErrorList{}
platform := config.Platform.AWS
allErrs = append(allErrs, validateServiceEndpoints(fldPath.Child("serviceEndpoints"), platform.Region, platform.ServiceEndpoints)...)
@@ -109,7 +110,7 @@ func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path,
}
if len(platform.VPC.Subnets) > 0 {
allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), platform.VPC.Subnets, networking, publish)...)
allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("vpc").Child("subnets"), config)...)
}
if platform.DefaultMachinePlatform != nil {
allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("defaultMachinePlatform"), platform, platform.DefaultMachinePlatform, controlPlaneReq, "", "")...)
@@ -218,67 +219,139 @@ func validatePublicIpv4Pool(ctx context.Context, meta *Metadata, fldPath *field.
return nil
}
func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, subnets []awstypes.Subnet, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList {
allErrs := field.ErrorList{}
privateSubnets, err := meta.PrivateSubnets(ctx)
if err != nil {
return append(allErrs, field.Invalid(fldPath, subnets, err.Error()))
// subnetData holds a subnet information collected from install config and AWS API for validations.
type subnetData struct {
// The subnet index in the install config.
Idx int
// The subnet assigned roles in the install config.
Roles []awstypes.SubnetRole
// The subnet metadata from AWS API.
Subnet
}
// subnetDataGroups is a collection of subnet information
// grouped by subnet type (i.e. public, private, and edge) and indexed by subnetIDs for validations.
type subnetDataGroups struct {
Public map[string]subnetData
Private map[string]subnetData
Edge map[string]subnetData
// A convenient alias that contains all information for all subnets.
All map[string]subnetData
}
// Converts subnetGroups (i.e. provided subnets) to subnetDataGroups to include additional information
// from the install-config such as index and roles for validations.
func (sdg *subnetDataGroups) From(ctx context.Context, meta *Metadata, providedSubnets []awstypes.Subnet) error {
if sdg.Private == nil {
sdg.Private = make(map[string]subnetData)
}
privateSubnetsIdx := map[string]int{}
for idx, subnet := range subnets {
if _, ok := privateSubnets[string(subnet.ID)]; ok {
privateSubnetsIdx[string(subnet.ID)] = idx
}
if sdg.Public == nil {
sdg.Public = make(map[string]subnetData)
}
if len(privateSubnets) == 0 {
allErrs = append(allErrs, field.Invalid(fldPath, subnets, "No private subnets found"))
if sdg.Edge == nil {
sdg.Edge = make(map[string]subnetData)
}
if sdg.All == nil {
sdg.All = make(map[string]subnetData)
}
publicSubnets, err := meta.PublicSubnets(ctx)
subnets, err := meta.Subnets(ctx)
if err != nil {
return append(allErrs, field.Invalid(fldPath, subnets, err.Error()))
return err
}
if publish == types.InternalPublishingStrategy && len(publicSubnets) > 0 {
logrus.Warnf("Public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy)
}
publicSubnetsIdx := map[string]int{}
for idx, subnet := range subnets {
if _, ok := publicSubnets[string(subnet.ID)]; ok {
publicSubnetsIdx[string(subnet.ID)] = idx
for idx, subnet := range providedSubnets {
var subnetDataGroup map[string]subnetData
var subnetMeta Subnet
if awsSubnet, ok := subnets.Private[string(subnet.ID)]; ok {
subnetDataGroup = sdg.Private
subnetMeta = awsSubnet
}
if awsSubnet, ok := subnets.Public[string(subnet.ID)]; ok {
subnetDataGroup = sdg.Public
subnetMeta = awsSubnet
}
if awsSubnet, ok := subnets.Edge[string(subnet.ID)]; ok {
subnetDataGroup = sdg.Edge
subnetMeta = awsSubnet
}
if subnetDataGroup == nil {
// Should not occur but safe against panics
continue
}
subnetData := subnetData{
Subnet: subnetMeta,
Idx: idx,
Roles: subnet.Roles,
}
subnetDataGroup[string(subnet.ID)] = subnetData
sdg.All[string(subnet.ID)] = subnetData
}
if len(publicSubnets) == 0 && awstypes.IsPublicOnlySubnetsEnabled() {
return nil
}
// validateSubnets ensures BYO subnets are valid.
func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList {
allErrs := field.ErrorList{}
networking := config.Networking
providedSubnets := config.AWS.VPC.Subnets
publish := config.Publish
subnetDataGroups := subnetDataGroups{}
if err := subnetDataGroups.From(ctx, meta, providedSubnets); err != nil {
return append(allErrs, field.Invalid(fldPath, providedSubnets, err.Error()))
}
publicOnlySubnet := awstypes.IsPublicOnlySubnetsEnabled()
if publicOnlySubnet && len(subnetDataGroups.Public) == 0 {
allErrs = append(allErrs, field.Required(fldPath, "public subnets are required for a public-only subnets cluster"))
}
edgeSubnets, err := meta.EdgeSubnets(ctx)
if err != nil {
return append(allErrs, field.Invalid(fldPath, subnets, err.Error()))
if !publicOnlySubnet && len(subnetDataGroups.Private) == 0 {
allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, "no private subnets found"))
}
edgeSubnetsIdx := map[string]int{}
for idx, subnet := range subnets {
if _, ok := edgeSubnets[string(subnet.ID)]; ok {
edgeSubnetsIdx[string(subnet.ID)] = idx
if publish == types.InternalPublishingStrategy && len(subnetDataGroups.Public) > 0 {
logrus.Warnf("public subnets should not be provided when publish is set to %s", types.InternalPublishingStrategy)
}
subnetsWithRole := make(map[awstypes.SubnetRoleType][]subnetData)
for _, subnet := range providedSubnets {
for _, role := range subnet.Roles {
subnetsWithRole[role.Type] = append(subnetsWithRole[role.Type], subnetDataGroups.All[string(subnet.ID)])
}
}
allErrs = append(allErrs, validateSubnetCIDR(fldPath, privateSubnets, privateSubnetsIdx, networking.MachineNetwork)...)
allErrs = append(allErrs, validateSubnetCIDR(fldPath, publicSubnets, publicSubnetsIdx, networking.MachineNetwork)...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, privateSubnets, privateSubnetsIdx, "private")...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, publicSubnets, publicSubnetsIdx, "public")...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, edgeSubnets, edgeSubnetsIdx, "edge")...)
allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Private, networking.MachineNetwork)...)
allErrs = append(allErrs, validateSubnetCIDR(fldPath, subnetDataGroups.Public, networking.MachineNetwork)...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Private, "private")...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Public, "public")...)
allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, subnetDataGroups.Edge, "edge")...)
if len(subnetsWithRole) > 0 {
allErrs = append(allErrs, validateSubnetRoles(fldPath, subnetsWithRole, subnetDataGroups, config)...)
} else {
allErrs = append(allErrs, validateUntaggedSubnets(ctx, fldPath, meta, subnetDataGroups)...)
}
privateZones := sets.New[string]()
publicZones := sets.New[string]()
for _, subnet := range privateSubnets {
for _, subnet := range subnetDataGroups.Private {
privateZones.Insert(subnet.Zone.Name)
}
for _, subnet := range publicSubnets {
for _, subnet := range subnetDataGroups.Public {
publicZones.Insert(subnet.Zone.Name)
}
if publish == types.ExternalPublishingStrategy && !publicZones.IsSuperset(privateZones) {
errMsg := fmt.Sprintf("No public subnet provided for zones %s", sets.List(privateZones.Difference(publicZones)))
allErrs = append(allErrs, field.Invalid(fldPath, subnets, errMsg))
allErrs = append(allErrs, field.Invalid(fldPath, providedSubnets, errMsg))
}
return allErrs
@@ -436,11 +509,11 @@ func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *fiel
return allErrs
}
func validateSubnetCIDR(fldPath *field.Path, subnets Subnets, idxMap map[string]int, networks []types.MachineNetworkEntry) field.ErrorList {
func validateSubnetCIDR(fldPath *field.Path, subnetDataGroup map[string]subnetData, networks []types.MachineNetworkEntry) field.ErrorList {
allErrs := field.ErrorList{}
for id, v := range subnets {
fp := fldPath.Index(idxMap[id])
cidr, _, err := net.ParseCIDR(v.CIDR)
for id, subnetData := range subnetDataGroup {
fp := fldPath.Index(subnetData.Idx)
cidr, _, err := net.ParseCIDR(subnetData.CIDR)
if err != nil {
allErrs = append(allErrs, field.Invalid(fp, id, err.Error()))
continue
@@ -459,27 +532,215 @@ func validateMachineNetworksContainIP(fldPath *field.Path, networks []types.Mach
return field.ErrorList{field.Invalid(fldPath, subnetName, fmt.Sprintf("subnet's CIDR range start %s is outside of the specified machine networks", ip))}
}
func validateDuplicateSubnetZones(fldPath *field.Path, subnets Subnets, idxMap map[string]int, typ string) field.ErrorList {
var keys []string
for id := range subnets {
keys = append(keys, id)
func validateDuplicateSubnetZones(fldPath *field.Path, subnetDataGroup map[string]subnetData, typ string) field.ErrorList {
subnetIDs := make([]string, 0)
for id := range subnetDataGroup {
subnetIDs = append(subnetIDs, id)
}
sort.Strings(keys)
sort.Strings(subnetIDs)
allErrs := field.ErrorList{}
zones := map[string]string{}
for _, id := range keys {
subnet := subnets[id]
if conflictingSubnet, ok := zones[subnet.Zone.Name]; ok {
errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnet.Zone.Name)
allErrs = append(allErrs, field.Invalid(fldPath.Index(idxMap[id]), id, errMsg))
for _, id := range subnetIDs {
subnetData := subnetDataGroup[id]
if conflictingSubnet, ok := zones[subnetData.Zone.Name]; ok {
errMsg := fmt.Sprintf("%s subnet %s is also in zone %s", typ, conflictingSubnet, subnetData.Zone.Name)
allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), id, errMsg))
} else {
zones[subnet.Zone.Name] = id
zones[subnetData.Zone.Name] = id
}
}
return allErrs
}
// validateSubnetRoles ensures BYO subnets have valid roles assigned if roles are provided.
func validateSubnetRoles(fldPath *field.Path, subnetsWithRole map[awstypes.SubnetRoleType][]subnetData, subnetDataGroups subnetDataGroups, config *types.InstallConfig) field.ErrorList {
allErrs := field.ErrorList{}
// BootstrapNode subnets must be assigned to public subnets
// in external cluster.
for _, bstrSubnet := range subnetsWithRole[awstypes.BootstrapNodeSubnetRole] {
// We validate edge subnets in subsequent validations.
if _, ok := subnetDataGroups.Edge[bstrSubnet.ID]; ok {
continue
}
if config.Publish == types.ExternalPublishingStrategy && !bstrSubnet.Public {
allErrs = append(allErrs, field.Invalid(fldPath.Index(bstrSubnet.Idx), bstrSubnet.ID,
fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", bstrSubnet.ID, awstypes.BootstrapNodeSubnetRole)))
}
}
// ClusterNode subnets must be assigned to private subnets
// unless cluster is public-only.
for _, cnSubnet := range subnetsWithRole[awstypes.ClusterNodeSubnetRole] {
// We validate edge subnets in subsequent validations.
if _, ok := subnetDataGroups.Edge[cnSubnet.ID]; ok {
continue
}
if cnSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() {
allErrs = append(allErrs, field.Invalid(fldPath.Index(cnSubnet.Idx), cnSubnet.ID,
fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", cnSubnet.ID, awstypes.ClusterNodeSubnetRole)))
}
}
// Type of ControlPlaneLB subnets must match its scope:
// - ControlPlaneInternalLB subnets must be private
// - ControlPlaneExternalLB subnets must be public.
// Private cluster must not have ControlPlaneExternalLB subnets (i.e. statically validated in pkg/types/aws/validation/platform.go).
for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneInternalLBSubnetRole] {
// We validate edge subnets in subsequent validations.
if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok {
continue
}
if ctrlPSubnet.Public && !awstypes.IsPublicOnlySubnetsEnabled() {
allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID,
fmt.Sprintf("subnet %s has role %s, but is public, expected to be private", ctrlPSubnet.ID, awstypes.ControlPlaneInternalLBSubnetRole)))
}
}
for _, ctrlPSubnet := range subnetsWithRole[awstypes.ControlPlaneExternalLBSubnetRole] {
// We validate edge subnets in subsequent validations.
if _, ok := subnetDataGroups.Edge[ctrlPSubnet.ID]; ok {
continue
}
if !ctrlPSubnet.Public {
allErrs = append(allErrs, field.Invalid(fldPath.Index(ctrlPSubnet.Idx), ctrlPSubnet.ID,
fmt.Sprintf("subnet %s has role %s, but is private, expected to be public", ctrlPSubnet.ID, awstypes.ControlPlaneExternalLBSubnetRole)))
}
}
// Type of IngressControllerLB subnets must match cluster scope:
// - In public cluster, only public IngressControllerLB subnets is allowed.
// - In private cluster, only private IngressControllerLB subnets is allowed.
for _, ingressSubnet := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] {
// We validate edge subnets in subsequent validations.
if _, ok := subnetDataGroups.Edge[ingressSubnet.ID]; ok {
continue
}
if ingressSubnet.Public != config.PublicIngress() {
subnetType := "private"
if ingressSubnet.Public {
subnetType = "public"
}
allErrs = append(allErrs, field.Invalid(fldPath.Index(ingressSubnet.Idx), ingressSubnet.ID,
fmt.Sprintf("subnet %s has role %s and is %s, which is not allowed when publish is set to %s", ingressSubnet.ID, awstypes.IngressControllerLBSubnetRole, subnetType, config.Publish)))
}
}
// IngressControllerLB subnets must be in different AZs as required by AWS CCM.
ingressZones := make(map[string]string)
for _, subnetData := range subnetsWithRole[awstypes.IngressControllerLBSubnetRole] {
if conflictingSubnet, ok := ingressZones[subnetData.Zone.Name]; ok {
allErrs = append(allErrs, field.Invalid(fldPath.Index(subnetData.Idx), subnetData.ID,
fmt.Sprintf("subnets %s and %s have role %s and are both in zone %s", conflictingSubnet, subnetData.ID, awstypes.IngressControllerLBSubnetRole, subnetData.Zone.Name)))
} else {
ingressZones[subnetData.Zone.Name] = subnetData.ID
}
}
// AZs of LB subnets match AZs of ClusterNode subnets.
lbRoles := []awstypes.SubnetRoleType{
awstypes.ControlPlaneInternalLBSubnetRole,
awstypes.IngressControllerLBSubnetRole,
}
if config.Publish == types.ExternalPublishingStrategy {
lbRoles = append(lbRoles, awstypes.ControlPlaneExternalLBSubnetRole)
}
for _, role := range lbRoles {
allErrs = append(allErrs, validateLBSubnetAZMatchClusterNodeAZ(fldPath, subnetDataGroups, role, subnetsWithRole[role], subnetsWithRole[awstypes.ClusterNodeSubnetRole])...)
}
// EdgeNode subnets must be subnets in Local or Wavelength Zones.
for _, edgeSubnet := range subnetsWithRole[awstypes.EdgeNodeSubnetRole] {
if _, ok := subnetDataGroups.Edge[edgeSubnet.ID]; !ok {
allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID,
fmt.Sprintf("subnet %s has role %s, but is not in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole)))
}
}
// Subnets that are in Local or Wavelength Zones must only have EdgeNode role.
for _, edgeSubnet := range subnetDataGroups.Edge {
for _, role := range edgeSubnet.Roles {
if role.Type != awstypes.EdgeNodeSubnetRole {
allErrs = append(allErrs, field.Invalid(fldPath.Index(edgeSubnet.Idx), edgeSubnet.ID,
fmt.Sprintf("subnet %s must only be assigned role %s since it is in a Local or WaveLength Zone", edgeSubnet.ID, awstypes.EdgeNodeSubnetRole)))
break
}
}
}
return allErrs
}
// validateUntaggedSubnets ensures there are no additional untagged subnets in the BYO VPC.
// An untagged subnet is a subnet without tag kubernetes.io/cluster/<cluster-id>.
// Untagged subnets may be selected by the CCM, leading to various bugs, RFEs, and support cases. See:
// - https://issues.redhat.com/browse/OCPBUGS-17432.
// - https://issues.redhat.com/browse/RFE-2816.
func validateUntaggedSubnets(ctx context.Context, fldPath *field.Path, meta *Metadata, subnetDataGroups subnetDataGroups) field.ErrorList {
allErrs := field.ErrorList{}
vpcSubnets, err := meta.VPCSubnets(ctx)
if err != nil {
return append(allErrs, field.Invalid(fldPath, meta.ProvidedSubnets, err.Error()))
}
untaggedSubnetIDs := make([]string, 0)
for _, subnet := range mergeSubnets(vpcSubnets.Public, vpcSubnets.Private, vpcSubnets.Edge) {
// We only check other subnets in the VPC that are not provided in the install-config.
if _, ok := subnetDataGroups.All[subnet.ID]; !ok && !subnet.Tags.HasTagKeyPrefix(TagNameKubernetesClusterPrefix) {
untaggedSubnetIDs = append(untaggedSubnetIDs, subnet.ID)
}
}
sort.Strings(untaggedSubnetIDs)
if len(untaggedSubnetIDs) > 0 {
errMsg := fmt.Sprintf("additional subnets %v without tag prefix %s are found in vpc %s of provided subnets. %s", untaggedSubnetIDs, TagNameKubernetesClusterPrefix, vpcSubnets.VPC,
fmt.Sprintf("Please add a tag %s to those subnets to exclude them from cluster installation or explicitly assign roles in the install-config to provided subnets", TagNameKubernetesUnmanaged))
allErrs = append(allErrs, field.Forbidden(fldPath, errMsg))
}
return allErrs
}
// validateLBSubnetAZMatchClusterNodeAZ ensures AZs of LB subnets match AZs of ClusterNode subnets.
// AWS load balancers will NOT register a node located in an AZ that is not enabled for the load balancer.
func validateLBSubnetAZMatchClusterNodeAZ(fldPath *field.Path, subnetDataGroups subnetDataGroups, lbType awstypes.SubnetRoleType, lbSubnets []subnetData, clusterNodeSubnets []subnetData) field.ErrorList {
allErrs := field.ErrorList{}
lbZoneSet := sets.New[string]()
for _, subnet := range lbSubnets {
// We validate edge subnets in another place.
if _, ok := subnetDataGroups.Edge[subnet.ID]; ok {
continue
}
lbZoneSet.Insert(subnet.Zone.Name)
}
nodeZoneSet := sets.New[string]()
for _, subnet := range clusterNodeSubnets {
// We validate edge subnets in another place.
if _, ok := subnetDataGroups.Edge[subnet.ID]; ok {
continue
}
nodeZoneSet.Insert(subnet.Zone.Name)
}
// If the nodes use an AZ that is not in load balancer enabled AZs,
// the router pod might be scheduled to nodes that the load balancer cannot reach.
if diffSet := nodeZoneSet.Difference(lbZoneSet); diffSet.Len() > 0 {
allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are not enabled for %s load balancers, nodes in those zones are unreachable", sets.List(diffSet), lbType)))
}
// If the load balancer includes an AZ that is not in node AZs,
// there will be no nodes in that AZ for the load balancer to register (i.e. not in use)
if diffSet := lbZoneSet.Difference(nodeZoneSet); diffSet.Len() > 0 {
allErrs = append(allErrs, field.Forbidden(fldPath, fmt.Sprintf("zones %v are enabled for %s load balancers, but are not used by any nodes", sets.List(diffSet), lbType)))
}
return allErrs
}
func validateServiceEndpoints(fldPath *field.Path, region string, services []awstypes.ServiceEndpoint) field.ErrorList {
allErrs := field.ErrorList{}
// Validate the endpoint overrides for all provided services.
@@ -527,17 +788,13 @@ func validateZoneLocal(ctx context.Context, meta *Metadata, fldPath *field.Path,
}
func validateEndpointAccessibility(endpointURL string) error {
// For each provided service endpoint, verify we can resolve and connect with net.Dial.
// Ignore e2e.local from unit tests.
if endpointURL == "e2e.local" {
return nil
if _, err := url.Parse(endpointURL); err != nil {
return fmt.Errorf("failed to parse service endpoint url: %w", err)
}
_, err := url.Parse(endpointURL)
if err != nil {
return err
if _, err := http.Head(endpointURL); err != nil { //nolint:gosec
return fmt.Errorf("failed to connect to service endpoint url: %w", err)
}
_, err = http.Head(endpointURL)
return err
return nil
}
var requiredServices = []string{

File diff suppressed because it is too large Load Diff