1
0
mirror of https://github.com/openshift/installer.git synced 2026-02-05 06:46:36 +01:00
Files
installer/upi/vsphere/upi.ps1
2024-04-11 14:50:05 -04:00

540 lines
23 KiB
PowerShell

#!/usr/bin/pwsh
. .\variables.ps1
. .\upi-functions.ps1
$ErrorActionPreference = "Stop"
# since we do not have ca for vsphere certs, we'll just set insecure
Set-PowerCLIConfiguration -InvalidCertificateAction:Ignore -Confirm:$false | Out-Null
$Env:GOVC_INSECURE = 1
# Connect to vCenter
Connect-VIServer -Server $vcenter -Credential (Import-Clixml $vcentercredpath)
if ($downloadInstaller) {
Write-Output "Downloading the most recent $($version) installer"
$releaseApiUri = "https://api.github.com/repos/openshift/okd/releases"
$progressPreference = 'silentlyContinue'
$webrequest = Invoke-WebRequest -uri $releaseApiUri
$progressPreference = 'Continue'
$releases = ConvertFrom-Json $webrequest.Content -AsHashtable
$publishedDate = (Get-Date).AddDays(-365)
$currentRelease = $null
foreach($r in $releases) {
if($r['name'] -like "*$($version)*") {
if ($publishedDate -lt $r['published_at'] ) {
$publishedDate = $r['published_at']
$currentRelease = $r
}
}
}
foreach($asset in $currentRelease['assets']) {
if($asset['name'] -like "openshift-install-linux*") {
$installerUrl = $asset['browser_download_url']
}
}
# If openshift-install doesn't exist on the path, download it and extract
if (-Not (Test-Path -Path "openshift-install")) {
$progressPreference = 'silentlyContinue'
Invoke-WebRequest -uri $installerUrl -OutFile "installer.tar.gz"
tar -xvf "installer.tar.gz"
$progressPreference = 'Continue'
}
}
if ($uploadTemplateOva) {
Write-Output "Downloading RHCOS OVA"
# If the OVA doesn't exist on the path, determine the url from openshift-install and download it.
if (-Not (Test-Path -Path "template-$($Version).ova")) {
Start-Process -Wait -Path ./openshift-install -ArgumentList @("coreos", "print-stream-json") -RedirectStandardOutput coreos.json
$coreosData = Get-Content -Path ./coreos.json | ConvertFrom-Json -AsHashtable
$ovaUri = $coreosData.architectures.x86_64.artifacts.vmware.formats.ova.disk.location
$progressPreference = 'silentlyContinue'
Invoke-WebRequest -uri $ovaUri -OutFile "template-$($Version).ova"
$progressPreference = 'Continue'
}
}
$sshKey = [string](Get-Content -Path $sshkeypath -Raw:$true) -Replace '\n',''
if ($createInstallConfig) {
# Without having to add additional powershell modules yaml is difficult to deal
# with. There is a supplied install-config.json which is converted to a powershell
# object
$config = ConvertFrom-Json -InputObject $installconfig
# Set the install-config.json from upi-variables
$config.metadata.name = $clustername
$config.baseDomain = $basedomain
$config.sshKey = $sshKey
$config.platform.vsphere.vcenter = $vcenter
$config.platform.vsphere.username = $username
$config.platform.vsphere.password = $password
$config.platform.vsphere.datacenter = $datacenter
$config.platform.vsphere.defaultDatastore = $datastore
$config.platform.vsphere.cluster = $cluster
$config.platform.vsphere.network = $portgroup
# $config.platform.vsphere.apiVIP = $apivip
# $config.platform.vsphere.ingressVIP = $ingressvip
$config.pullSecret = $pullsecret -replace "`n", "" -replace " ", ""
# Write out the install-config.yaml (really json)
$config | ConvertTo-Json -Depth 8 | Out-File -FilePath install-config.yaml -Force:$true
}
if ($generateIgnitions) {
# openshift-install create manifests
start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "manifests")
# Remove master machines and the worker machinesets
rm -f openshift/99_openshift-cluster-api_master-machines-*.yaml openshift/99_openshift-cluster-api_worker-machineset-*.yaml
# openshift-install create ignition-configs
start-process -Wait -FilePath ./openshift-install -argumentlist @("create", "ignition-configs")
}
# Check failure domains. If not set, create a default single failure domain from settings
if ($null -eq $failure_domains) {
Write-Output "Generating Failure Domain..."
$failure_domains = @"
[
{
"datacenter": "$($datacenter)",
"cluster": "$($cluster)",
"datastore": "$($datastore)",
"network": "$($portgroup)"
}
]
"@
}
$fds = $failure_domains | ConvertFrom-Json
# Convert the installer metadata to a powershell object
$metadata = Get-Content -Path ./metadata.json | ConvertFrom-Json
# Since we are using MachineSets for the workers make sure we set the
# template name to what is expected to be generated by the installer.
if ($null -eq $vm_template) {
$vm_template = "$( $metadata.infraID )-rhcos"
}
# Create tag for all resources we create
$tagCategory = Get-TagCategory -Name "openshift-$($metadata.infraID)" -ErrorAction continue 2>$null
if (-Not $?) {
Write-Output "Creating Tag Category openshift-$($metadata.infraID)"
$tagCategory = New-TagCategory -Name "openshift-$($metadata.infraID)" -EntityType "urn:vim25:VirtualMachine","urn:vim25:ResourcePool","urn:vim25:Folder","urn:vim25:Datastore","urn:vim25:StoragePod"
}
$tag = Get-Tag -Category $tagCategory -Name "$($metadata.infraID)" -ErrorAction continue 2>$null
if (-Not $?) {
Write-Output "Creating Tag $($metadata.infraID)"
$tag = New-Tag -Category $tagCategory -Name "$($metadata.infraID)"
}
# Check each failure domain for ova template
foreach ($fd in $fds)
{
$datastoreInfo = Get-Datastore -Name $fd.datastore -Location $fd.datacenter
# If the folder already exists
Write-Output "Checking for folder in failure domain $($fd.datacenter)/$($fd.cluster)"
$folder = Get-Folder -Name $metadata.infraID -Location $fd.datacenter -ErrorAction continue 2>$null
# Otherwise create the folder within the datacenter as defined in the upi-variables
if (-Not $?) {
Write-Output "Creating folder $($metadata.infraID) in datacenter $($fd.datacenter)"
(get-view (Get-Datacenter -Name $fd.datacenter).ExtensionData.vmfolder).CreateFolder($metadata.infraID)
$folder = Get-Folder -Name $metadata.infraID -Location $fd.datacenter
New-TagAssignment -Entity $folder -Tag $tag > $null
}
# If the rhcos virtual machine already exists
Write-Output "Checking for vm template in failure domain $($fd.datacenter)/$($fd.cluster)"
$template = Get-VM -Name $vm_template -Location $fd.datacenter -ErrorAction continue
# Otherwise import the ova to a random host on the vSphere cluster
if (-Not$?)
{
$vmhost = Get-Random -InputObject (Get-VMHost -Location (Get-Cluster $fd.cluster))
$ovfConfig = Get-OvfConfiguration -Ovf "template-$( $Version ).ova"
$ovfConfig.NetworkMapping.VM_Network.Value = $portgroup
$template = Import-Vapp -Source "template-$( $Version ).ova" -Name $vm_template -OvfConfiguration $ovfConfig -VMHost $vmhost -Datastore $datastoreInfo -InventoryLocation $folder -Force:$true
$templateVIObj = Get-View -VIObject $template.Name
# Need to look into upgrading hardware. For me it keeps throwing exception.
<# try {
$templateVIObj.UpgradeVM($hardwareVersion)
}
catch {
Write-Output "Something happened setting VM hardware version"
Write-Output $_
} #>
New-TagAssignment -Entity $template -Tag $tag
Set-VM -VM $template -MemoryGB 16 -NumCpu 4 -CoresPerSocket 4 -Confirm:$false > $null
Get-HardDisk -VM $template | Select-Object -First 1 | Set-HardDisk -CapacityGB 120 -Confirm:$false > $null
New-AdvancedSetting -Entity $template -name "disk.EnableUUID" -value 'TRUE' -confirm:$false -Force > $null
New-AdvancedSetting -Entity $template -name "guestinfo.ignition.config.data.encoding" -value "base64" -confirm:$false -Force > $null
#$snapshot = New-Snapshot -VM $template -Name "linked-clone" -Description "linked-clone" -Memory -Quiesce
}
}
Write-Output "Creating LB"
# Data needed for LB VM creation
$rp = Get-Cluster -Name $fds[0].cluster -Server $vcenter
$datastoreInfo = Get-Datastore -Name $fds[0].datastore -Server $vcenter -Location $fds[0].datacenter
$folder = Get-Folder -Name $metadata.infraID -Location $fds[0].datacenter
$template = Get-VM -Name $vm_template -Location $fds[0].datacenter
# Create LB for Cluster
$ignition = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes((New-LoadBalancerIgnition $sshKey)))
$network = New-VMNetworkConfig -Hostname "$($metadata.infraID)-lb" -IPAddress $lb_ip_address -Netmask $netmask -Gateway $gateway -DNS $dns -Network $failure_domains[0].network
$vm = New-OpenShiftVM -IgnitionData $ignition -Name "$($metadata.infraID)-lb" -Template $template -ResourcePool $rp -Datastore $datastoreInfo -Location $folder -Tag $tag -Networking $network -Network $($fds[0].network) -MemoryMB 8192 -NumCpu 4
$vm | Start-VM
# Take the $virtualmachines defined in upi-variables and convert to a powershell object
if ($null -eq $virtualmachines)
{
$virtualmachines = New-VMConfigs
}
$vmHash = ConvertFrom-Json -InputObject $virtualmachines -AsHashtable
Write-Progress -id 222 -Activity "Creating virtual machines" -PercentComplete 0
$vmStep = (100 / $vmHash.virtualmachines.Count)
$vmCount = 1
foreach ($key in $vmHash.virtualmachines.Keys) {
$node = $vmHash.virtualmachines[$key]
$name = "$($metadata.infraID)-$($key)"
Write-Output "Creating $($name)"
$rp = Get-Cluster -Name $node.cluster -Server $node.server
##$datastore = Get-Datastore -Name $node.datastore -Server $node.server
$datastoreInfo = Get-Datastore -Name $node.datastore -Location $node.datacenter
# Pull network config for each node
if ($node.type -eq "master") {
$numCPU = $control_plane_num_cpus
$memory = $control_plane_memory
} elseif ($node.type -eq "worker") {
$numCPU = $compute_num_cpus
$memory = $compute_memory
} else {
# should only be bootstrap
$numCPU = $control_plane_num_cpus
$memory = $control_plane_memory
}
$ip = $node.ip
$network = New-VMNetworkConfig -Hostname $name -IPAddress $ip -Netmask $netmask -Gateway $gateway -DNS $dns
# Get the content of the ignition file per machine type (bootstrap, master, worker)
$bytes = Get-Content -Path "./$($node.type).ign" -AsByteStream
$ignition = [Convert]::ToBase64String($bytes)
# Get correct template / folder
$folder = Get-Folder -Name $metadata.infraID -Location $node.datacenter
$template = Get-VM -Name $vm_template -Location $($node.datacenter)
# Clone the virtual machine from the imported template
#$vm = New-OpenShiftVM -Template $template -Name $name -ResourcePool $rp -Datastore $datastoreInfo -Location $folder -LinkedClone -ReferenceSnapshot $snapshot -IgnitionData $ignition -Tag $tag -Networking $network -NumCPU $numCPU -MemoryMB $memory
$vm = New-OpenShiftVM -Template $template -Name $name -ResourcePool $rp -Datastore $datastoreInfo -Location $folder -IgnitionData $ignition -Tag $tag -Networking $network -Network $node.network -NumCPU $numCPU -MemoryMB $memory
# Assign tag so we can later clean up
# New-TagAssignment -Entity $vm -Tag $tag
# New-AdvancedSetting -Entity $vm -name "guestinfo.ignition.config.data" -value $ignition -confirm:$false -Force > $null
# New-AdvancedSetting -Entity $vm -name "guestinfo.hostname" -value $name -Confirm:$false -Force > $null
if ($node.type -eq "master" -And $delayVMStart) {
# To give bootstrap some time to start, lets wait 2 minutes
Start-ThreadJob -ThrottleLimit 5 -InputObject $vm {
Start-Sleep -Seconds 90
$input | Start-VM
}
} elseif ($node.type -eq "worker" -And $delayVMStart) {
# Workers are not needed right away, gotta wait till masters
# have started machine-server. wait 7 minutes to start.
Start-ThreadJob -ThrottleLimit 5 -InputObject $vm {
Start-Sleep -Seconds 600
$input | Start-VM
}
}
else {
$vm | Start-VM
}
Write-Progress -id 222 -Activity "Creating virtual machines" -PercentComplete ($vmStep * $vmCount)
$vmCount++
}
Write-Progress -id 222 -Activity "Completed virtual machines" -PercentComplete 100 -Completed
## This is nice to have to clear screen when doing things manually. Maybe i'll
# make this configurable.
# Clear-Host
# Instead of restarting openshift-install to wait for bootstrap, monitor
# the bootstrap configmap in the kube-system namespace
# Extract the Client Certificate Data from auth/kubeconfig
$match = Select-String "client-certificate-data: (.*)" -Path ./auth/kubeconfig
[Byte[]]$bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value)
$clientCertData = [System.Text.Encoding]::ASCII.GetString($bytes)
# Extract the Client Key Data from auth/kubeconfig
$match = Select-String "client-key-data: (.*)" -Path ./auth/kubeconfig
$bytes = [Convert]::FromBase64String($match.Matches.Groups[1].Value)
$clientKeyData = [System.Text.Encoding]::ASCII.GetString($bytes)
# Create a X509Certificate2 object for Invoke-WebRequest
$cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::CreateFromPem($clientCertData, $clientKeyData)
# Extract the kubernetes endpoint uri
$match = Select-String "server: (.*)" -Path ./auth/kubeconfig
$kubeurl = $match.Matches.Groups[1].Value
if ($waitForComplete)
{
$apiTimeout = (20*60)
$apiCount = 1
$apiSleep = 30
Write-Progress -Id 444 -Status "1% Complete" -Activity "API" -PercentComplete 1
:api while ($true) {
Start-Sleep -Seconds $apiSleep
try {
$webrequest = Invoke-WebRequest -Uri "$($kubeurl)/version" -SkipCertificateCheck
$version = (ConvertFrom-Json $webrequest.Content).gitVersion
if ($version -ne "" ) {
Write-Debug "API Version: $($version)"
Write-Progress -Id 444 -Status "Completed" -Activity "API" -PercentComplete 100
break api
}
}
catch {}
$percentage = ((($apiCount*$apiSleep)/$apiTimeout)*100)
if ($percentage -le 100) {
Write-Progress -Id 444 -Status "$percentage% Complete" -Activity "API" -PercentComplete $percentage
}
$apiCount++
}
$bootstrapTimeout = (30*60)
$bootstrapCount = 1
$bootstrapSleep = 30
Write-Progress -Id 333 -Status "1% Complete" -Activity "Bootstrap" -PercentComplete 1
:bootstrap while ($true)
{
Start-Sleep -Seconds $bootstrapSleep
try
{
$webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/namespaces/kube-system/configmaps/bootstrap" -SkipCertificateCheck
$bootstrapStatus = (ConvertFrom-Json $webrequest.Content).data.status
if ($bootstrapStatus -eq "complete")
{
Get-VM "$( $metadata.infraID )-bootstrap" | Stop-VM -Confirm:$false | Remove-VM -DeletePermanently -Confirm:$false
Write-Progress -Id 333 -Status "Completed" -Activity "Bootstrap" -PercentComplete 100
break bootstrap
}
}
catch
{
}
$percentage = ((($bootstrapCount*$bootstrapSleep)/$bootstrapTimeout)*100)
if ($percentage -le 100)
{
Write-Progress -Id 333 -Status "$percentage% Complete" -Activity "Bootstrap" -PercentComplete $percentage
}
else
{
Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow
}
$bootstrapCount++
}
# Now that bootstrap is complete, we should be getting worker node CSRs that need to be approved before being
# able to finish installation. Lets monitor for CSRs, approve them and verify the number of worker nodes have
# now appeared and are Ready before moving on.
# [ngirard@fedora ibm7-installs]$ oc get csr | grep Pending
#csr-2hgbd 2m52s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending
#csr-lwmgf 2m19s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending
#csr-scvk6 2m30s kubernetes.io/kube-apiserver-client-kubelet system:serviceaccount:openshift-machine-config-operator:node-bootstrapper <none> Pending
# apis/certificates.k8s.io/v1/certificatesigningrequests
$csrTimeout = (600/5)
$csrCount = 1
$csrSleep = 5
Write-Progress -Id 222 -Status "1% Complete" -Activity "Worker Ready" -PercentComplete 0
:csrLoop while ($true)
{
Start-Sleep -Seconds $csrSleep
try
{
$webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests" -SkipCertificateCheck
$csrs = (ConvertFrom-Json $webrequest.Content).items
foreach ($csr in $csrs)
{
# Check if no status (Pending) and if its a type we are looking for (kubernetes.io/kubelet-serving) (kubernetes.io/kube-apiserver-client-kubelet)
$bootstrapper = "system:serviceaccount:openshift-machine-config-operator:node-bootstrapper"
$nodeUser = "system:node:"
$csrUser = $csr.spec.username
if ($csr.status.conditions -eq $null -And ($csrUser -eq $bootstrapper -Or $csrUser.Contains($nodeUser)))
{
$conditions = New-Object System.Collections.ArrayList
$condition = @{
type = "Approved"
status = "True"
reason = "PowershellApproved"
message = "This CSR was approved by script in PowerShell."
}
$conditions.Add($condition) > $null
$csr.status | add-member -Name "conditions" -value $conditions -MemberType NoteProperty
Write-Output "Accepting CSR: $( $csr.metadata.name )"
$csrResponse = Invoke-RestMethod -Method "Put" -Certificate $cert -Uri "$( $kubeurl )/apis/certificates.k8s.io/v1/certificatesigningrequests/$( $csr.metadata.name )/approval" -SkipCertificateCheck -Body (ConvertTo-Json $csr -Depth 6)
}
}
}
catch
{
#Write-Output $_
}
# Check number of worker nodes with NotReady/Ready. NotReady will be 1 pt where Ready will be 2.
$currentComputePoints = 0
try
{
$webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/api/v1/nodes" -SkipCertificateCheck
$nodes = (ConvertFrom-Json $webrequest.Content).items
foreach ($node in $nodes)
{
if ($node.metadata.labels.psobject.properties.name -Contains "node-role.kubernetes.io/worker")
{
#Write-Output "Checking node $($node.metadata.name)"
foreach ($condition in $node.status.conditions)
{
if ($condition.type -eq "Ready")
{
#Write-Output "Is node ready? $($condition.status)"
if ($condition.status -eq "True")
{
$currentComputePoints = $currentComputePoints + 2
}
else
{
$currentComputePoints++
}
}
}
}
}
}
catch
{
#Write-Output $_
}
$maxComputePoints = $compute_count * 2
$percentage = ((($currentComputePoints)/$maxComputePoints)*100)
if ($percentage -eq 100)
{
Write-Progress -Id 222 -Status "Completed" -Activity "Worker Ready" -PercentComplete 100
break csrLoop
}
elseif ($percentage -le 100)
{
Write-Progress -Id 222 -Status "$percentage% Complete" -Activity "Worker Ready" -PercentComplete $percentage
}
if ($csrCount -ge $csrTimeout)
{
Write-Output "Warning: Bootstrap taking longer than usual." -NoNewLine -ForegroundColor Yellow
break csrLoop
}
$csrCount++
}
$progressMsg = ""
Write-Progress -Id 111 -Status "1% Complete" -Activity "Install" -PercentComplete 1
:installcomplete while ($true)
{
Start-Sleep -Seconds 30
try
{
$webrequest = Invoke-WebRequest -Certificate $cert -Uri "$( $kubeurl )/apis/config.openshift.io/v1/clusterversions" -SkipCertificateCheck
$clusterversions = ConvertFrom-Json $webrequest.Content -AsHashtable
# just like the installer check the status conditions of the clusterversions config
foreach ($condition in $clusterversions['items'][0]['status']['conditions'])
{
switch ($condition['type'])
{
"Progressing" {
if ($condition['status'] -eq "True")
{
$matchper = ($condition['message'] | Select-String "^Working.*\(([0-9]{1,3})\%.*\)")
$matchmsg = ($condition['message'] | Select-String -AllMatches -Pattern "^(Working.*)\:.*")
# During install, the status of CVO will / may go degraded due to operators going
# degraded from taking a while to install. It seems this is the new norm as control
# plane takes a while to roll out and certain operators go degraded until the control
# plane is stable.
if ($matchmsg.Matches.Groups -ne $null)
{
$progressMsg = $matchmsg.Matches.Groups[1].Value
$progressPercent = $matchper.Matches.Groups[1].Value
Write-Progress -Id 111 -Status "$progressPercent% Complete - $( $progressMsg )" -Activity "Install" -PercentComplete $progressPercent
}
continue
}
}
"Available" {
if ($condition['status'] -eq "True")
{
Write-Progress -Id 111 -Activity "Install" -Status "Completed" -PercentComplete 100
break installcomplete
}
continue
}
Default {
continue
}
}
}
}
catch
{
Write-Output "Unable to check operators"
Write-Output $_
}
}
}
Get-Job | Remove-Job
Write-Output "Install Complete!"