From 0055065143ee52ee6252cd7896d551e374500a59 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Tue, 16 Jul 2019 10:05:02 -0400 Subject: [PATCH] baremetal: add baremetal IPI platform This adds experimental support for the baremetal IPI platform. Baremetal IPI is implemented via a libvirt bootstrap VM, and an Ironic instance that handles provisioning of baremetal nodes. This baremetal platform is still experimental and relies on the openshift-metal3/dev-scripts to perform a complete deployment. Co-authored-by: Antoni Segura Puimedon Co-authored-by: Ben Nemec Co-authored-by: Derek Higgins Co-authored-by: Eduardo Minguez Perez Co-authored-by: Mark McLoughlin Co-authored-by: Russell Bryant Co-authored-by: Sandhya Dasu Co-authored-by: Stephen Benjamin Co-authored-by: Steven Hardy Co-authored-by: Yolanda Robla --- README.md | 3 +- cmd/openshift-install/destroy.go | 1 + docs/user/metal/README.md | 22 ++ docs/user/metal/install_ipi.md | 225 ++++++++++++++++++ images/baremetal/Dockerfile.ci | 24 ++ pkg/asset/cluster/baremetal/OWNERS | 5 + pkg/asset/cluster/baremetal/baremetal.go | 16 ++ pkg/asset/cluster/metadata.go | 4 + pkg/asset/cluster/tfvars.go | 19 ++ pkg/asset/ignition/machine/node.go | 13 +- pkg/asset/installconfig/installconfig.go | 1 + pkg/asset/installconfig/platformcredscheck.go | 3 +- pkg/asset/machines/baremetal/OWNERS | 5 + pkg/asset/machines/baremetal/machines.go | 71 ++++++ pkg/asset/machines/baremetal/machinesets.go | 79 ++++++ pkg/asset/machines/master.go | 15 ++ pkg/asset/machines/worker.go | 23 +- pkg/asset/manifests/cloudproviderconfig.go | 3 +- pkg/asset/manifests/dns.go | 3 +- pkg/asset/manifests/infrastructure.go | 3 + pkg/asset/rhcos/image.go | 3 + pkg/asset/tls/mcscertkey.go | 11 +- pkg/destroy/baremetal/OWNERS | 5 + pkg/destroy/baremetal/baremetal.go | 43 ++++ pkg/destroy/baremetal/doc.go | 2 + pkg/destroy/baremetal/register.go | 10 + pkg/tfvars/baremetal/OWNERS | 5 + pkg/tfvars/baremetal/baremetal.go | 110 +++++++++ pkg/tfvars/libvirt/cache.go | 6 + pkg/types/baremetal/OWNERS | 5 + pkg/types/baremetal/defaults/platform.go | 67 ++++++ pkg/types/baremetal/doc.go | 6 + pkg/types/baremetal/machinepool.go | 13 + pkg/types/baremetal/metadata.go | 7 + pkg/types/baremetal/platform.go | 66 +++++ pkg/types/baremetal/validation/machinepool.go | 12 + pkg/types/baremetal/validation/platform.go | 44 ++++ pkg/types/clustermetadata.go | 5 + pkg/types/defaults/installconfig.go | 3 + pkg/types/installconfig.go | 8 + pkg/types/machinepools.go | 6 + pkg/types/validation/installconfig.go | 7 + pkg/types/validation/installconfig_test.go | 6 +- pkg/types/validation/machinepools.go | 5 + pkg/validate/validate.go | 23 ++ 45 files changed, 1006 insertions(+), 10 deletions(-) create mode 100644 docs/user/metal/README.md create mode 100644 docs/user/metal/install_ipi.md create mode 100644 images/baremetal/Dockerfile.ci create mode 100644 pkg/asset/cluster/baremetal/OWNERS create mode 100644 pkg/asset/cluster/baremetal/baremetal.go create mode 100644 pkg/asset/machines/baremetal/OWNERS create mode 100644 pkg/asset/machines/baremetal/machines.go create mode 100644 pkg/asset/machines/baremetal/machinesets.go create mode 100644 pkg/destroy/baremetal/OWNERS create mode 100644 pkg/destroy/baremetal/baremetal.go create mode 100644 pkg/destroy/baremetal/doc.go create mode 100644 pkg/destroy/baremetal/register.go create mode 100644 pkg/tfvars/baremetal/OWNERS create mode 100644 pkg/tfvars/baremetal/baremetal.go create mode 100644 pkg/types/baremetal/OWNERS create mode 100644 pkg/types/baremetal/defaults/platform.go create mode 100644 pkg/types/baremetal/doc.go create mode 100644 pkg/types/baremetal/machinepool.go create mode 100644 pkg/types/baremetal/metadata.go create mode 100644 pkg/types/baremetal/platform.go create mode 100644 pkg/types/baremetal/validation/machinepool.go create mode 100644 pkg/types/baremetal/validation/platform.go diff --git a/README.md b/README.md index 61d5c2f98c..c7fc329956 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ ## Supported Platforms * [AWS](docs/user/aws/README.md) -* [Bare-metal](docs/user/metal/install_upi.md) +* [Bare Metal (UPI)](docs/user/metal/install_upi.md) +* [Bare Metal (IPI) (Experimental)](docs/user/metal/install_ipi.md) * [Libvirt with KVM](docs/dev/libvirt/README.md) (development only) * [OpenStack (experimental)](docs/user/openstack/README.md) * [vSphere](docs/user/vsphere/install_upi.md) diff --git a/cmd/openshift-install/destroy.go b/cmd/openshift-install/destroy.go index f56918d02e..a9fb56c0aa 100644 --- a/cmd/openshift-install/destroy.go +++ b/cmd/openshift-install/destroy.go @@ -9,6 +9,7 @@ import ( "github.com/openshift/installer/pkg/destroy" _ "github.com/openshift/installer/pkg/destroy/aws" _ "github.com/openshift/installer/pkg/destroy/azure" + _ "github.com/openshift/installer/pkg/destroy/baremetal" "github.com/openshift/installer/pkg/destroy/bootstrap" _ "github.com/openshift/installer/pkg/destroy/gcp" _ "github.com/openshift/installer/pkg/destroy/libvirt" diff --git a/docs/user/metal/README.md b/docs/user/metal/README.md new file mode 100644 index 0000000000..04984f0741 --- /dev/null +++ b/docs/user/metal/README.md @@ -0,0 +1,22 @@ +# Support for Bare Metal Environments + +OpenShift has support for bare metal deployments with either [User +provided infrastructure (UPI)](install_upi.md), or [Installer-provided +instrastructure (IPI)](install_ipi.md). + +The following is a summary of key differences: + +* UPI bare metal + * Provisioning hosts is an external requirement + * Requires extra DNS configuration + * Requires setup of load balancers + * Offers more control and choice over infrastructure + +* IPI bare metal + * Has built-in hardware provisioning components, will provision nodes with RHCOS automatically, + and supports the Machine API for ongoing management of these hosts. + * Automates internal DNS requirements + * Automates setup of self-hosted load balancers + * Supports “openshift-install create cluster” for bare metal environments + using this infrastructure automation, but requires the use of compatible + hardware, as described in [install_ipi.md](install_ipi.md). diff --git a/docs/user/metal/install_ipi.md b/docs/user/metal/install_ipi.md new file mode 100644 index 0000000000..610086a67f --- /dev/null +++ b/docs/user/metal/install_ipi.md @@ -0,0 +1,225 @@ +# Bare Metal IPI (Installer Provisioned Infrastructure) Overview + +Current Status: **Experimental** + +This document discusses the installer support for an IPI (Installer Provisioned +Infrastructure) install for bare metal hosts. This includes platform support +for the management of bare metal hosts, as well as some automation of DNS and +load balancing to bring up the cluster. + +The upstream project that provides Kubernetes-native management of bare metal +hosts is [metal3.io](http://metal3.io). + +For UPI (User Provisioned Infrastructure) based instructions for bare metal +deployments, see [install_upi.md](install_upi.md). + +## Prerequisites + +### Ironic + +Currently, the `baremetal` platform requires an existing Ironic environment. +This will eventually be handled by `openshift-install`, with Ironic being +deployed onto the bootstrap node. Until then, users of the `baremetal` platform +should use the +[openshift-metal3/dev-scripts](https://github.com/openshift-metal3/dev-scripts) +repository to handle configuration of Ironic. + +The following PR contains the WIP changes for automating Ironic from +`openshift-install`: https://github.com/openshift-metal3/kni-installer/pull/100 + +### Network Requirements + +It is assumed that all hosts have at least 2 NICs, used for the following +purposes: + +* **NIC #1 - External Network** + * This network is the main network used by the cluster, including API traffic + and application traffic. + * ***DHCP*** + * External DHCP is assumed on this network. It is **strongly** recommended + to set up DHCP reservations for each of the hosts in the cluster to + ensure that they retain stable IP addresses. + * A pool of dynamic addresses should also be available on this network, as + the provisioning host and temporary bootstrap VM will also need addresses + on this network. + * ***NTP*** + * A time source must be accessible from this network. + * ***Reserved VIPs (Virtual IPs)*** - 3 IP addresses must be reserved on this + network for use by the cluster. Specifically, these IPs will serve the + following purposes: + * API - This IP will be used to reach the cluster API. + * Ingress - This IP will be used by cluster ingress traffic + * DNS - This IP will be used internally by the cluster for automating + internal DNS requirements. + * ***External DNS*** - While the cluster automates the internal DNS + requirements, two external DNS records must be created in whatever DNS + server is appropriate for this environment. + * `api..` - pointing to the API VIP + * `*.apps..` - pointing to the Ingress VIP + +* **NIC #2 - Provisioning Network** + * A private, non-routed network, used for PXE based provisioning. + * DHCP is automated for this network. + * Addressing for this network is currently hard coded as `172.22.0.0/24`, but + will be made configurable in the future. + +* **Out-of-band Management Network** + * Servers will typically have an additional NIC used by the onboard + management controllers (BMCs). These BMCs must be accessible and routed to + the host. + +### Provisioning Host + +The installer must be run from a host that is attached to the same networks as +the cluster, as described in the previous section. We refer to this host as +the *provisioning host*. The easiest way to provide a provisioning host is to +use one of the hosts that is intended to later become a worker node in the same +cluster. That way it is already connected to the proper networks. + +It is recommended that the provisioning host be a bare metal host, as it must be +able to use libvirt to launch the OpenShift bootstrap VM locally. + +### Supported Hardware + +The architecture is intended to support a wide variety of hardware. This was +one of the reasons Ironic is used as an underlying technology. However, so far +development and testing has focused on PXE based provisioning using IPMI for +out-of-band management of hosts. Other provisioning approaches will be added, +tested, and documented over time. + +## Installation Process + +Once an environment has been prepared according to the documented +pre-requisites, the install process is the same as other IPI based platforms. + +`openshift-install create cluster` + +However, it is recommended to prepare an `install-config.yaml` file in advance, +containing all of the details of the bare metal hosts to be provisioned. + +### Install Config + +The `install-config.yaml` file requires some additional details. Most of the +information is teaching the installer and the resulting cluster enough about +the available hardware so that it is able to fully manage it. + +Here is an example `install-config.yaml` with the required `baremetal` platform +details. + +**IMPORTANT NOTE:** The current install configuration for the `baremetal` +platform should be considered experimental and still subject to change without +backwards compatibility. In particular, some items likely to change soon +include: + +* The `image` section will get completely removed. + +* The `hardwareProfile` is currently exposed as a way to allow specifying + different hardware parameters for deployment. By default, we will deploy + RHCOS to the first disk, but that may not be appropriate for all hardware. + The `hardwareProfile` is the field we have available to change that. This + interface is subject to change. In the meantime, hardware profiles can be + found here: + https://github.com/metal3-io/baremetal-operator/blob/master/pkg/hardware/profile.go#L48 + +```yaml +apiVersion: v1beta4 +baseDomain: test.metalkube.org +metadata: + name: ostest +compute: +- name: worker + replicas: 1 +controlPlane: + name: master + replicas: 3 + platform: + baremetal: {} +platform: + baremetal: + apiVIP: 192.168.111.5 + hosts: + - name: openshift-master-0 + role: master + bmc: + address: ipmi://192.168.111.1:6230 + username: admin + password: password + bootMACAddress: 00:11:07:4e:f6:68 + hardwareProfile: default + - name: openshift-master-1 + role: master + bmc: + address: ipmi://192.168.111.1:6231 + username: admin + password: password + bootMACAddress: 00:11:07:4e:f6:6c + hardwareProfile: default + - name: openshift-master-2 + role: master + bmc: + address: ipmi://192.168.111.1:6232 + username: admin + password: password + bootMACAddress: 00:11:07:4e:f6:70 + hardwareProfile: default + - name: openshift-worker-0 + role: master + bmc: + address: ipmi://192.168.111.1:6233 + username: admin + password: password + bootMACAddress: 00:11:07:4e:f6:71 + hardwareProfile: default + image: + source: "http://172.22.0.1/images/rhcos-ootpa-latest.qcow2" + checksum: 2b3b1e19e18627d89da400b63430d5bb + deployKernel: http://172.22.0.1/images/ironic-python-agent.kernel + deployRamdisk: http://172.22.0.1/images/ironic-python-agent.initramfs +pullSecret: ... +sshKey: ... +``` + +## Work in Progress + +Integration of the `baremetal` platform is still a work-in-progress across +various parts of OpenShift. This section discusses key items that are not yet +fully integrated, and their workarounds. + +Note that once this work moves into the `openshift/installer` repository, new +issues will get created or existing issues will be moved to track these gaps +instead of the leaving the existing issues against the KNI fork of the installer. + +### Deployment of the `baremetal-operator` + +The `baremetal-operator` provides the server side of the API used by the +`baremetal` platform `Machine` actuator +([cluster-api-provider-baremetal](https://github.com/metal3-io/cluster-api-provider-baremetal)). +This is currently handled by the +[08_deploy_bmo.sh](https://github.com/openshift-metal3/dev-scripts/blob/master/08_deploy_bmo.sh) +script. + +This will be replaced by `machine-api-operator` integration and the following +PR: https://github.com/openshift/machine-api-operator/pull/302 + +### `BareMetalHost` registration by the Installer + +`openshift-install` needs to create the `BareMetalHost` objects that represent +the inventory of hardware under management. This is currently handled by the +[11_register_hosts.sh](https://github.com/openshift-metal3/dev-scripts/blob/master/11_register_hosts.sh) +script. + +https://github.com/openshift-metal3/kni-installer/issues/46 + +### `destroy cluster` support + +`openshift-install destroy cluster` is not supported for the `baremetal` +platform. + +https://github.com/openshift-metal3/kni-installer/issues/74 + +### install gather not supported + +When an installation fails, `openshift-install` will attempt to gather debug +information from hosts. This is not yet supported by the `baremetal` platform. + +https://github.com/openshift-metal3/kni-installer/issues/79 diff --git a/images/baremetal/Dockerfile.ci b/images/baremetal/Dockerfile.ci new file mode 100644 index 0000000000..8fc266c231 --- /dev/null +++ b/images/baremetal/Dockerfile.ci @@ -0,0 +1,24 @@ +# This Dockerfile is a used by CI to publish an installer image +# It builds an image containing only the openshift-install. + +FROM registry.svc.ci.openshift.org/openshift/release:golang-1.10 AS builder +WORKDIR /go/src/github.com/openshift/installer +COPY . . +RUN TAGS="libvirt baremetal" hack/build.sh + + +FROM registry.svc.ci.openshift.org/origin/4.1:base +COPY --from=builder /go/src/github.com/openshift/installer/bin/openshift-install /bin/openshift-install + +RUN yum install --setopt=tsflags=nodocs -y \ + yum update -y && \ + yum install --setopt=tsflags=nodocs -y \ + libvirt-libs && \ + yum clean all && rm -rf /var/cache/yum/* + +RUN mkdir /output && chown 1000:1000 /output +USER 1000:1000 +ENV PATH /bin +ENV HOME /output +WORKDIR /output +ENTRYPOINT ["/bin/openshift-install"] diff --git a/pkg/asset/cluster/baremetal/OWNERS b/pkg/asset/cluster/baremetal/OWNERS new file mode 100644 index 0000000000..11af0bba91 --- /dev/null +++ b/pkg/asset/cluster/baremetal/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md +# This file just uses aliases defined in OWNERS_ALIASES. + +reviewers: + - baremetal-reviewers diff --git a/pkg/asset/cluster/baremetal/baremetal.go b/pkg/asset/cluster/baremetal/baremetal.go new file mode 100644 index 0000000000..54c42accf8 --- /dev/null +++ b/pkg/asset/cluster/baremetal/baremetal.go @@ -0,0 +1,16 @@ +// Package baremetal extracts bare metal metadata from install +// configurations. +package baremetal + +import ( + "github.com/openshift/installer/pkg/types" + "github.com/openshift/installer/pkg/types/baremetal" +) + +// Metadata converts an install configuration to bare metal metadata. +func Metadata(config *types.InstallConfig) *baremetal.Metadata { + return &baremetal.Metadata{ + LibvirtURI: config.Platform.BareMetal.LibvirtURI, + IronicURI: config.Platform.BareMetal.IronicURI, + } +} diff --git a/pkg/asset/cluster/metadata.go b/pkg/asset/cluster/metadata.go index 7cfbd20a84..76c95546b0 100644 --- a/pkg/asset/cluster/metadata.go +++ b/pkg/asset/cluster/metadata.go @@ -8,6 +8,7 @@ import ( "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/cluster/aws" "github.com/openshift/installer/pkg/asset/cluster/azure" + "github.com/openshift/installer/pkg/asset/cluster/baremetal" "github.com/openshift/installer/pkg/asset/cluster/gcp" "github.com/openshift/installer/pkg/asset/cluster/libvirt" "github.com/openshift/installer/pkg/asset/cluster/openstack" @@ -15,6 +16,7 @@ import ( "github.com/openshift/installer/pkg/types" awstypes "github.com/openshift/installer/pkg/types/aws" azuretypes "github.com/openshift/installer/pkg/types/azure" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" gcptypes "github.com/openshift/installer/pkg/types/gcp" libvirttypes "github.com/openshift/installer/pkg/types/libvirt" nonetypes "github.com/openshift/installer/pkg/types/none" @@ -71,6 +73,8 @@ func (m *Metadata) Generate(parents asset.Parents) (err error) { metadata.ClusterPlatformMetadata.Azure = azure.Metadata(installConfig.Config) case gcptypes.Name: metadata.ClusterPlatformMetadata.GCP = gcp.Metadata(installConfig.Config) + case baremetaltypes.Name: + metadata.ClusterPlatformMetadata.BareMetal = baremetal.Metadata(installConfig.Config) case nonetypes.Name, vspheretypes.Name: default: return errors.Errorf("no known platform") diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index 5770bc3e10..d0090a4caf 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -28,11 +28,13 @@ import ( "github.com/openshift/installer/pkg/tfvars" awstfvars "github.com/openshift/installer/pkg/tfvars/aws" azuretfvars "github.com/openshift/installer/pkg/tfvars/azure" + baremetaltfvars "github.com/openshift/installer/pkg/tfvars/baremetal" gcptfvars "github.com/openshift/installer/pkg/tfvars/gcp" libvirttfvars "github.com/openshift/installer/pkg/tfvars/libvirt" openstacktfvars "github.com/openshift/installer/pkg/tfvars/openstack" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/none" @@ -255,6 +257,23 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { Filename: fmt.Sprintf(TfPlatformVarsFileName, platform), Data: data, }) + case baremetal.Name: + data, err = baremetaltfvars.TFVars( + installConfig.Config.Platform.BareMetal.LibvirtURI, + installConfig.Config.Platform.BareMetal.IronicURI, + string(*rhcosImage), + "baremetal", + "provisioning", + installConfig.Config.Platform.BareMetal.Hosts, + installConfig.Config.Platform.BareMetal.Image, + ) + if err != nil { + return errors.Wrapf(err, "failed to get %s Terraform variables", platform) + } + t.FileList = append(t.FileList, &asset.File{ + Filename: fmt.Sprintf(TfPlatformVarsFileName, platform), + Data: data, + }) default: logrus.Warnf("unrecognized platform %s", platform) } diff --git a/pkg/asset/ignition/machine/node.go b/pkg/asset/ignition/machine/node.go index 04f7b0f2ac..d1930d2555 100644 --- a/pkg/asset/ignition/machine/node.go +++ b/pkg/asset/ignition/machine/node.go @@ -8,11 +8,22 @@ import ( "github.com/vincent-petithory/dataurl" "github.com/openshift/installer/pkg/types" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" ) // pointerIgnitionConfig generates a config which references the remote config // served by the machine config server. func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, role string) *ignition.Config { + var ignitionHost string + switch installConfig.Platform.Name() { + case baremetaltypes.Name: + // Baremetal needs to point directly at the VIP because we don't have a + // way to configure DNS before Ignition runs. + ignitionHost = fmt.Sprintf("%s:22623", installConfig.BareMetal.APIVIP) + default: + ignitionHost = fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain()) + } + return &ignition.Config{ Ignition: ignition.Ignition{ Version: ignition.MaxVersion.String(), @@ -21,7 +32,7 @@ func pointerIgnitionConfig(installConfig *types.InstallConfig, rootCA []byte, ro Source: func() *url.URL { return &url.URL{ Scheme: "https", - Host: fmt.Sprintf("api-int.%s:22623", installConfig.ClusterDomain()), + Host: ignitionHost, Path: fmt.Sprintf("/config/%s", role), } }().String(), diff --git a/pkg/asset/installconfig/installconfig.go b/pkg/asset/installconfig/installconfig.go index f7e8903ace..2c8f3cfc25 100644 --- a/pkg/asset/installconfig/installconfig.go +++ b/pkg/asset/installconfig/installconfig.go @@ -73,6 +73,7 @@ func (a *InstallConfig) Generate(parents asset.Parents) error { a.Config.VSphere = platform.VSphere a.Config.Azure = platform.Azure a.Config.GCP = platform.GCP + a.Config.BareMetal = platform.BareMetal if err := a.setDefaults(); err != nil { return errors.Wrap(err, "failed to set defaults for install config") diff --git a/pkg/asset/installconfig/platformcredscheck.go b/pkg/asset/installconfig/platformcredscheck.go index 5a9e72145b..62b3e58d77 100644 --- a/pkg/asset/installconfig/platformcredscheck.go +++ b/pkg/asset/installconfig/platformcredscheck.go @@ -11,6 +11,7 @@ import ( gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/none" @@ -59,7 +60,7 @@ func (a *PlatformCredsCheck) Generate(dependencies asset.Parents) error { opts := new(clientconfig.ClientOpts) opts.Cloud = ic.Config.Platform.OpenStack.Cloud _, err = clientconfig.GetCloudFromYAML(opts) - case libvirt.Name, none.Name, vsphere.Name: + case baremetal.Name, libvirt.Name, none.Name, vsphere.Name: // no creds to check case azure.Name: _, err = azureconfig.GetSession() diff --git a/pkg/asset/machines/baremetal/OWNERS b/pkg/asset/machines/baremetal/OWNERS new file mode 100644 index 0000000000..11af0bba91 --- /dev/null +++ b/pkg/asset/machines/baremetal/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md +# This file just uses aliases defined in OWNERS_ALIASES. + +reviewers: + - baremetal-reviewers diff --git a/pkg/asset/machines/baremetal/machines.go b/pkg/asset/machines/baremetal/machines.go new file mode 100644 index 0000000000..e6ad580842 --- /dev/null +++ b/pkg/asset/machines/baremetal/machines.go @@ -0,0 +1,71 @@ +// Package baremetal generates Machine objects for bare metal. +package baremetal + +import ( + "fmt" + + baremetalprovider "github.com/metal3-io/cluster-api-provider-baremetal/pkg/apis/baremetal/v1alpha1" + + machineapi "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openshift/installer/pkg/types" + "github.com/openshift/installer/pkg/types/baremetal" +) + +// Machines returns a list of machines for a machinepool. +func Machines(clusterID string, config *types.InstallConfig, pool *types.MachinePool, role, userDataSecret string) ([]machineapi.Machine, error) { + if configPlatform := config.Platform.Name(); configPlatform != baremetal.Name { + return nil, fmt.Errorf("non bare metal configuration: %q", configPlatform) + } + if poolPlatform := pool.Platform.Name(); poolPlatform != baremetal.Name { + return nil, fmt.Errorf("non bare metal machine-pool: %q", poolPlatform) + } + clustername := config.ObjectMeta.Name + platform := config.Platform.BareMetal + + total := int64(1) + if pool.Replicas != nil { + total = *pool.Replicas + } + provider := provider(clustername, config.Networking.MachineCIDR.String(), platform, userDataSecret) + var machines []machineapi.Machine + for idx := int64(0); idx < total; idx++ { + machine := machineapi.Machine{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "machine.openshift.io/v1beta1", + Kind: "Machine", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "openshift-machine-api", + Name: fmt.Sprintf("%s-%s-%d", clustername, pool.Name, idx), + Labels: map[string]string{ + "machine.openshift.io/cluster-api-cluster": clustername, + "machine.openshift.io/cluster-api-machine-role": role, + "machine.openshift.io/cluster-api-machine-type": role, + }, + }, + Spec: machineapi.MachineSpec{ + ProviderSpec: machineapi.ProviderSpec{ + Value: &runtime.RawExtension{Object: provider}, + }, + // we don't need to set Versions, because we control those via cluster operators. + }, + } + machines = append(machines, machine) + } + + return machines, nil +} + +func provider(clusterName string, networkInterfaceAddress string, platform *baremetal.Platform, userDataSecret string) *baremetalprovider.BareMetalMachineProviderSpec { + return &baremetalprovider.BareMetalMachineProviderSpec{ + Image: baremetalprovider.Image{ + URL: platform.Image.Source, + Checksum: platform.Image.Checksum, + }, + UserData: &corev1.SecretReference{Name: userDataSecret}, + } +} diff --git a/pkg/asset/machines/baremetal/machinesets.go b/pkg/asset/machines/baremetal/machinesets.go new file mode 100644 index 0000000000..aa226fa61e --- /dev/null +++ b/pkg/asset/machines/baremetal/machinesets.go @@ -0,0 +1,79 @@ +// Package baremetal generates Machine objects for bare metal. +package baremetal + +import ( + "fmt" + + machineapi "github.com/openshift/cluster-api/pkg/apis/machine/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" + + "github.com/openshift/installer/pkg/types" + "github.com/openshift/installer/pkg/types/baremetal" +) + +// MachineSets returns a list of machinesets for a machinepool. +func MachineSets(clusterID string, config *types.InstallConfig, pool *types.MachinePool, role, userDataSecret string) ([]*machineapi.MachineSet, error) { + if configPlatform := config.Platform.Name(); configPlatform != baremetal.Name { + return nil, fmt.Errorf("non bare metal configuration: %q", configPlatform) + } + // FIXME: empty is a valid case for bare metal as we don't use it? + if poolPlatform := pool.Platform.Name(); poolPlatform != "" && poolPlatform != baremetal.Name { + return nil, fmt.Errorf("non bare metal machine-pool: %q", poolPlatform) + } + clustername := config.ObjectMeta.Name + platform := config.Platform.BareMetal + // FIXME: bare metal actuator does not support any options from machinepool. + // mpool := pool.Platform.BareMetal + + total := int64(0) + if pool.Replicas != nil { + total = *pool.Replicas + } + + provider := provider(clustername, config.Networking.MachineCIDR.String(), platform, userDataSecret) + name := fmt.Sprintf("%s-%s-%d", clustername, pool.Name, 0) + mset := &machineapi.MachineSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "machine.openshift.io/v1beta1", + Kind: "MachineSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "openshift-machine-api", + Name: name, + Labels: map[string]string{ + "machine.openshift.io/cluster-api-cluster": clustername, + "machine.openshift.io/cluster-api-machine-role": role, + "machine.openshift.io/cluster-api-machine-type": role, + }, + }, + Spec: machineapi.MachineSetSpec{ + Replicas: pointer.Int32Ptr(int32(total)), + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "machine.openshift.io/cluster-api-machineset": name, + "machine.openshift.io/cluster-api-cluster": clustername, + }, + }, + Template: machineapi.MachineTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "machine.openshift.io/cluster-api-machineset": name, + "machine.openshift.io/cluster-api-cluster": clustername, + "machine.openshift.io/cluster-api-machine-role": role, + "machine.openshift.io/cluster-api-machine-type": role, + }, + }, + Spec: machineapi.MachineSpec{ + ProviderSpec: machineapi.ProviderSpec{ + Value: &runtime.RawExtension{Object: provider}, + }, + // we don't need to set Versions, because we control those via cluster operators. + }, + }, + }, + } + + return []*machineapi.MachineSet{mset}, nil +} diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go index 91fb32f649..97ae3b35cc 100644 --- a/pkg/asset/machines/master.go +++ b/pkg/asset/machines/master.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/ghodss/yaml" + baremetalapi "github.com/metal3-io/cluster-api-provider-baremetal/pkg/apis" + baremetalprovider "github.com/metal3-io/cluster-api-provider-baremetal/pkg/apis/baremetal/v1alpha1" gcpapi "github.com/openshift/cluster-api-provider-gcp/pkg/apis" gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1" libvirtapi "github.com/openshift/cluster-api-provider-libvirt/pkg/apis" @@ -27,6 +29,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/machines/aws" "github.com/openshift/installer/pkg/asset/machines/azure" + "github.com/openshift/installer/pkg/asset/machines/baremetal" "github.com/openshift/installer/pkg/asset/machines/gcp" "github.com/openshift/installer/pkg/asset/machines/libvirt" "github.com/openshift/installer/pkg/asset/machines/machineconfig" @@ -37,6 +40,7 @@ import ( awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults" azuretypes "github.com/openshift/installer/pkg/types/azure" azuredefaults "github.com/openshift/installer/pkg/types/azure/defaults" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" gcptypes "github.com/openshift/installer/pkg/types/gcp" libvirttypes "github.com/openshift/installer/pkg/types/libvirt" nonetypes "github.com/openshift/installer/pkg/types/none" @@ -186,6 +190,15 @@ func (m *Master) Generate(dependencies asset.Parents) error { return errors.Wrap(err, "failed to create master machine objects") } azure.ConfigMasters(machines, clusterID.InfraID) + case baremetaltypes.Name: + mpool := defaultBareMetalMachinePoolPlatform() + mpool.Set(ic.Platform.BareMetal.DefaultMachinePlatform) + mpool.Set(pool.Platform.BareMetal) + pool.Platform.BareMetal = &mpool + machines, err = baremetal.Machines(clusterID.InfraID, ic, pool, "master", "master-user-data") + if err != nil { + return errors.Wrap(err, "failed to create master machine objects") + } case nonetypes.Name, vspheretypes.Name: default: return fmt.Errorf("invalid Platform") @@ -273,12 +286,14 @@ func (m *Master) Machines() ([]machineapi.Machine, error) { scheme := runtime.NewScheme() awsapi.AddToScheme(scheme) azureapi.AddToScheme(scheme) + baremetalapi.AddToScheme(scheme) gcpapi.AddToScheme(scheme) libvirtapi.AddToScheme(scheme) openstackapi.AddToScheme(scheme) decoder := serializer.NewCodecFactory(scheme).UniversalDecoder( awsprovider.SchemeGroupVersion, azureprovider.SchemeGroupVersion, + baremetalprovider.SchemeGroupVersion, gcpprovider.SchemeGroupVersion, libvirtprovider.SchemeGroupVersion, openstackprovider.SchemeGroupVersion, diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index f86248a766..e2d14911e0 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -6,6 +6,8 @@ import ( "path/filepath" "github.com/ghodss/yaml" + baremetalapi "github.com/metal3-io/cluster-api-provider-baremetal/pkg/apis" + baremetalprovider "github.com/metal3-io/cluster-api-provider-baremetal/pkg/apis/baremetal/v1alpha1" gcpapi "github.com/openshift/cluster-api-provider-gcp/pkg/apis" gcpprovider "github.com/openshift/cluster-api-provider-gcp/pkg/apis/gcpprovider/v1beta1" libvirtapi "github.com/openshift/cluster-api-provider-libvirt/pkg/apis" @@ -27,6 +29,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/machines/aws" "github.com/openshift/installer/pkg/asset/machines/azure" + "github.com/openshift/installer/pkg/asset/machines/baremetal" "github.com/openshift/installer/pkg/asset/machines/gcp" "github.com/openshift/installer/pkg/asset/machines/libvirt" "github.com/openshift/installer/pkg/asset/machines/machineconfig" @@ -37,6 +40,7 @@ import ( awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults" azuretypes "github.com/openshift/installer/pkg/types/azure" azuredefaults "github.com/openshift/installer/pkg/types/azure/defaults" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" gcptypes "github.com/openshift/installer/pkg/types/gcp" libvirttypes "github.com/openshift/installer/pkg/types/libvirt" nonetypes "github.com/openshift/installer/pkg/types/none" @@ -91,6 +95,10 @@ func defaultOpenStackMachinePoolPlatform(flavor string) openstacktypes.MachinePo } } +func defaultBareMetalMachinePoolPlatform() baremetaltypes.MachinePool { + return baremetaltypes.MachinePool{} +} + // Worker generates the machinesets for `worker` machine pool. type Worker struct { UserDataFile *asset.File @@ -191,6 +199,18 @@ func (w *Worker) Generate(dependencies asset.Parents) error { for _, set := range sets { machineSets = append(machineSets, set) } + case baremetaltypes.Name: + mpool := defaultBareMetalMachinePoolPlatform() + mpool.Set(ic.Platform.BareMetal.DefaultMachinePlatform) + mpool.Set(pool.Platform.BareMetal) + pool.Platform.BareMetal = &mpool + sets, err := baremetal.MachineSets(clusterID.InfraID, ic, &pool, "worker", "worker-user-data") + if err != nil { + return errors.Wrap(err, "failed to create worker machine objects") + } + for _, set := range sets { + machineSets = append(machineSets, set) + } case gcptypes.Name: mpool := defaultGCPMachinePoolPlatform() mpool.Set(ic.Platform.GCP.DefaultMachinePlatform) @@ -235,7 +255,6 @@ func (w *Worker) Generate(dependencies asset.Parents) error { for _, set := range sets { machineSets = append(machineSets, set) } - case nonetypes.Name, vspheretypes.Name: default: return fmt.Errorf("invalid Platform") @@ -315,12 +334,14 @@ func (w *Worker) MachineSets() ([]machineapi.MachineSet, error) { scheme := runtime.NewScheme() awsapi.AddToScheme(scheme) azureapi.AddToScheme(scheme) + baremetalapi.AddToScheme(scheme) gcpapi.AddToScheme(scheme) libvirtapi.AddToScheme(scheme) openstackapi.AddToScheme(scheme) decoder := serializer.NewCodecFactory(scheme).UniversalDecoder( awsprovider.SchemeGroupVersion, azureprovider.SchemeGroupVersion, + baremetalprovider.SchemeGroupVersion, gcpprovider.SchemeGroupVersion, libvirtprovider.SchemeGroupVersion, openstackprovider.SchemeGroupVersion, diff --git a/pkg/asset/manifests/cloudproviderconfig.go b/pkg/asset/manifests/cloudproviderconfig.go index 4a69094528..a35253d3b7 100644 --- a/pkg/asset/manifests/cloudproviderconfig.go +++ b/pkg/asset/manifests/cloudproviderconfig.go @@ -17,6 +17,7 @@ import ( vspheremanifests "github.com/openshift/installer/pkg/asset/manifests/vsphere" awstypes "github.com/openshift/installer/pkg/types/aws" azuretypes "github.com/openshift/installer/pkg/types/azure" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" gcptypes "github.com/openshift/installer/pkg/types/gcp" libvirttypes "github.com/openshift/installer/pkg/types/libvirt" nonetypes "github.com/openshift/installer/pkg/types/none" @@ -78,7 +79,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error { } switch installConfig.Config.Platform.Name() { - case awstypes.Name, libvirttypes.Name, nonetypes.Name: + case awstypes.Name, libvirttypes.Name, nonetypes.Name, baremetaltypes.Name: return nil case openstacktypes.Name: cm.Data[cloudProviderConfigDataKey] = openstackmanifests.CloudProviderConfig() diff --git a/pkg/asset/manifests/dns.go b/pkg/asset/manifests/dns.go index d0d29d49f3..94b6bfd01c 100644 --- a/pkg/asset/manifests/dns.go +++ b/pkg/asset/manifests/dns.go @@ -19,6 +19,7 @@ import ( icgcp "github.com/openshift/installer/pkg/asset/installconfig/gcp" awstypes "github.com/openshift/installer/pkg/types/aws" azuretypes "github.com/openshift/installer/pkg/types/azure" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" gcptypes "github.com/openshift/installer/pkg/types/gcp" libvirttypes "github.com/openshift/installer/pkg/types/libvirt" nonetypes "github.com/openshift/installer/pkg/types/none" @@ -106,7 +107,7 @@ func (d *DNS) Generate(dependencies asset.Parents) error { } config.Spec.PublicZone = &configv1.DNSZone{ID: zone.Name} config.Spec.PrivateZone = &configv1.DNSZone{ID: fmt.Sprintf("%s-private-zone", clusterID.InfraID)} - case libvirttypes.Name, openstacktypes.Name, nonetypes.Name, vspheretypes.Name: + case libvirttypes.Name, openstacktypes.Name, baremetaltypes.Name, nonetypes.Name, vspheretypes.Name: default: return errors.New("invalid Platform") } diff --git a/pkg/asset/manifests/infrastructure.go b/pkg/asset/manifests/infrastructure.go index 09967643c3..202eab420c 100644 --- a/pkg/asset/manifests/infrastructure.go +++ b/pkg/asset/manifests/infrastructure.go @@ -14,6 +14,7 @@ import ( "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/none" @@ -84,6 +85,8 @@ func (i *Infrastructure) Generate(dependencies asset.Parents) error { config.Status.PlatformStatus.Azure = &configv1.AzurePlatformStatus{ ResourceGroupName: fmt.Sprintf("%s-rg", clusterID.InfraID), } + case baremetal.Name: + config.Status.PlatformStatus.Type = configv1.BareMetalPlatformType case gcp.Name: config.Status.PlatformStatus.Type = configv1.GCPPlatformType config.Status.PlatformStatus.GCP = &configv1.GCPPlatformStatus{ diff --git a/pkg/asset/rhcos/image.go b/pkg/asset/rhcos/image.go index 81a8f8ceb0..85f786abb5 100644 --- a/pkg/asset/rhcos/image.go +++ b/pkg/asset/rhcos/image.go @@ -14,6 +14,7 @@ import ( "github.com/openshift/installer/pkg/rhcos" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/none" @@ -67,6 +68,8 @@ func (i *Image) Generate(p asset.Parents) error { case azure.Name: //TODO(serbrech): change to right image once available. osimage = "/resourceGroups/rhcos_images/providers/Microsoft.Compute/images/rhcostestimage" + case baremetal.Name: + osimage, err = rhcos.QEMU(ctx) case none.Name, vsphere.Name: default: return errors.New("invalid Platform") diff --git a/pkg/asset/tls/mcscertkey.go b/pkg/asset/tls/mcscertkey.go index 8d6f8015a8..cd0a5c79b9 100644 --- a/pkg/asset/tls/mcscertkey.go +++ b/pkg/asset/tls/mcscertkey.go @@ -3,9 +3,11 @@ package tls import ( "crypto/x509" "crypto/x509/pkix" + "net" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" + baremetaltypes "github.com/openshift/installer/pkg/types/baremetal" ) // MCSCertKey is the asset that generates the MCS key/cert pair. @@ -37,7 +39,14 @@ func (a *MCSCertKey) Generate(dependencies asset.Parents) error { Subject: pkix.Name{CommonName: hostname}, ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, Validity: ValidityTenYears, - DNSNames: []string{hostname}, + } + + switch installConfig.Config.Platform.Name() { + case baremetaltypes.Name: + cfg.IPAddresses = []net.IP{net.ParseIP(installConfig.Config.BareMetal.APIVIP)} + cfg.DNSNames = []string{hostname, installConfig.Config.BareMetal.APIVIP} + default: + cfg.DNSNames = []string{hostname} } return a.SignedCertKey.Generate(cfg, ca, "machine-config-server", DoNotAppendParent) diff --git a/pkg/destroy/baremetal/OWNERS b/pkg/destroy/baremetal/OWNERS new file mode 100644 index 0000000000..11af0bba91 --- /dev/null +++ b/pkg/destroy/baremetal/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md +# This file just uses aliases defined in OWNERS_ALIASES. + +reviewers: + - baremetal-reviewers diff --git a/pkg/destroy/baremetal/baremetal.go b/pkg/destroy/baremetal/baremetal.go new file mode 100644 index 0000000000..cf00d3ac00 --- /dev/null +++ b/pkg/destroy/baremetal/baremetal.go @@ -0,0 +1,43 @@ +// +build baremetal + +package baremetal + +import ( + "github.com/libvirt/libvirt-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/openshift/installer/pkg/destroy/providers" + "github.com/openshift/installer/pkg/types" +) + +// ClusterUninstaller holds the various options for the cluster we want to delete. +type ClusterUninstaller struct { + LibvirtURI string + IronicURI string + Logger logrus.FieldLogger +} + +// Run is the entrypoint to start the uninstall process. +func (o *ClusterUninstaller) Run() error { + o.Logger.Debug("Deleting bare metal resources") + + // FIXME: close the connection + _, err := libvirt.NewConnect(o.LibvirtURI) + if err != nil { + return errors.Wrap(err, "failed to connect to Libvirt daemon") + } + + o.Logger.Debug("FIXME: delete resources!") + + return nil +} + +// New returns bare metal Uninstaller from ClusterMetadata. +func New(logger logrus.FieldLogger, metadata *types.ClusterMetadata) (providers.Destroyer, error) { + return &ClusterUninstaller{ + LibvirtURI: metadata.ClusterPlatformMetadata.BareMetal.LibvirtURI, + IronicURI: metadata.ClusterPlatformMetadata.BareMetal.IronicURI, + Logger: logger, + }, nil +} diff --git a/pkg/destroy/baremetal/doc.go b/pkg/destroy/baremetal/doc.go new file mode 100644 index 0000000000..e51819329a --- /dev/null +++ b/pkg/destroy/baremetal/doc.go @@ -0,0 +1,2 @@ +// Package baremetal provides a cluster-destroyer for bare metal clusters. +package baremetal diff --git a/pkg/destroy/baremetal/register.go b/pkg/destroy/baremetal/register.go new file mode 100644 index 0000000000..12ccff422e --- /dev/null +++ b/pkg/destroy/baremetal/register.go @@ -0,0 +1,10 @@ +// +build baremetal + +// Package baremetal provides a cluster-destroyer for bare metal clusters. +package baremetal + +import "github.com/openshift/installer/pkg/destroy/providers" + +func init() { + providers.Registry["baremetal"] = New +} diff --git a/pkg/tfvars/baremetal/OWNERS b/pkg/tfvars/baremetal/OWNERS new file mode 100644 index 0000000000..11af0bba91 --- /dev/null +++ b/pkg/tfvars/baremetal/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md +# This file just uses aliases defined in OWNERS_ALIASES. + +reviewers: + - baremetal-reviewers diff --git a/pkg/tfvars/baremetal/baremetal.go b/pkg/tfvars/baremetal/baremetal.go new file mode 100644 index 0000000000..021c6c47ee --- /dev/null +++ b/pkg/tfvars/baremetal/baremetal.go @@ -0,0 +1,110 @@ +// Package baremetal contains bare metal specific Terraform-variable logic. +package baremetal + +import ( + "encoding/json" + "github.com/metal3-io/baremetal-operator/pkg/bmc" + "github.com/metal3-io/baremetal-operator/pkg/hardware" + libvirttfvars "github.com/openshift/installer/pkg/tfvars/libvirt" + "github.com/openshift/installer/pkg/types/baremetal" + "github.com/pkg/errors" +) + +type config struct { + LibvirtURI string `json:"libvirt_uri,omitempty"` + IronicURI string `json:"ironic_uri,omitempty"` + Image string `json:"os_image,omitempty"` + ExternalBridge string `json:"external_bridge,omitempty"` + ProvisioningBridge string `json:"provisioning_bridge,omitempty"` + + // Data required for control plane deployment - several maps per host, because of terraform's limitations + Hosts []map[string]interface{} `json:"hosts"` + RootDevices []map[string]interface{} `json:"root_devices"` + Properties []map[string]interface{} `json:"properties"` + DriverInfos []map[string]interface{} `json:"driver_infos"` + InstanceInfos []map[string]interface{} `json:"instance_infos"` +} + +// TFVars generates bare metal specific Terraform variables. +func TFVars(libvirtURI, ironicURI, osImage, externalBridge, provisioningBridge string, platformHosts []*baremetal.Host, image baremetal.Image) ([]byte, error) { + osImage, err := libvirttfvars.CachedImage(osImage) + if err != nil { + return nil, errors.Wrap(err, "failed to use cached libvirt image") + } + + var hosts, rootDevices, properties, driverInfos, instanceInfos []map[string]interface{} + + for _, host := range platformHosts { + // Get hardware profile + if host.HardwareProfile == "default" { + host.HardwareProfile = hardware.DefaultProfileName + } + + profile, err := hardware.GetProfile(host.HardwareProfile) + if err != nil { + return nil, err + } + + // BMC Driver Info + accessDetails, err := bmc.NewAccessDetails(host.BMC.Address) + if err != nil { + return nil, err + } + credentials := bmc.Credentials{ + Username: host.BMC.Username, + Password: host.BMC.Password, + } + driverInfo := accessDetails.DriverInfo(credentials) + driverInfo["deploy_kernel"] = image.DeployKernel + driverInfo["deploy_ramdisk"] = image.DeployRamdisk + + // Host Details + hostMap := map[string]interface{}{ + "name": host.Name, + "port_address": host.BootMACAddress, + "driver": accessDetails.Type(), + } + + // Properties + propertiesMap := map[string]interface{}{ + "local_gb": profile.LocalGB, + "cpu_arch": profile.CPUArch, + } + + // Root device hints + rootDevice := make(map[string]interface{}) + if profile.RootDeviceHints.HCTL != "" { + rootDevice["hctl"] = profile.RootDeviceHints.HCTL + } else { + rootDevice["name"] = profile.RootDeviceHints.DeviceName + } + + // Instance Info + instanceInfo := map[string]interface{}{ + "root_gb": 25, // FIXME(stbenjam): Needed until https://storyboard.openstack.org/#!/story/2005165 + "image_source": image.Source, + "image_checksum": image.Checksum, + } + + hosts = append(hosts, hostMap) + properties = append(properties, propertiesMap) + driverInfos = append(driverInfos, driverInfo) + rootDevices = append(rootDevices, rootDevice) + instanceInfos = append(instanceInfos, instanceInfo) + } + + cfg := &config{ + LibvirtURI: libvirtURI, + IronicURI: ironicURI, + Image: osImage, + ExternalBridge: externalBridge, + ProvisioningBridge: provisioningBridge, + Hosts: hosts, + Properties: properties, + DriverInfos: driverInfos, + RootDevices: rootDevices, + InstanceInfos: instanceInfos, + } + + return json.MarshalIndent(cfg, "", " ") +} diff --git a/pkg/tfvars/libvirt/cache.go b/pkg/tfvars/libvirt/cache.go index 3912d175ec..afcd19e194 100644 --- a/pkg/tfvars/libvirt/cache.go +++ b/pkg/tfvars/libvirt/cache.go @@ -14,6 +14,12 @@ import ( "golang.org/x/sys/unix" ) +// CachedImage returns the location of the cached image. +// FIXME: Exported for use by baremetal platform. +func CachedImage(uri string) (string, error) { + return cachedImage(uri) +} + // cachedImage leaves non-file:// image URIs unalterered. // Other URIs are retrieved with a local cache at // $XDG_CACHE_HOME/openshift-install/libvirt [1]. This allows you to diff --git a/pkg/types/baremetal/OWNERS b/pkg/types/baremetal/OWNERS new file mode 100644 index 0000000000..11af0bba91 --- /dev/null +++ b/pkg/types/baremetal/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md +# This file just uses aliases defined in OWNERS_ALIASES. + +reviewers: + - baremetal-reviewers diff --git a/pkg/types/baremetal/defaults/platform.go b/pkg/types/baremetal/defaults/platform.go new file mode 100644 index 0000000000..d51ac98f52 --- /dev/null +++ b/pkg/types/baremetal/defaults/platform.go @@ -0,0 +1,67 @@ +package defaults + +import ( + "fmt" + "net" + + "github.com/openshift/installer/pkg/types" + "github.com/openshift/installer/pkg/types/baremetal" +) + +// Defaults for the baremetal platform. +const ( + LibvirtURI = "qemu:///system" + IronicURI = "http://localhost:6385/v1" + ExternalBridge = "baremetal" + ProvisioningBridge = "provisioning" + HardwareProfile = "default" + APIVIP = "" + IngressVIP = "" +) + +// SetPlatformDefaults sets the defaults for the platform. +func SetPlatformDefaults(p *baremetal.Platform, c *types.InstallConfig) { + if p.LibvirtURI == "" { + p.LibvirtURI = LibvirtURI + } + + if p.IronicURI == "" { + p.IronicURI = IronicURI + } + + if p.ExternalBridge == "" { + p.ExternalBridge = ExternalBridge + } + + if p.ProvisioningBridge == "" { + p.ProvisioningBridge = ProvisioningBridge + } + + for _, host := range p.Hosts { + if host.HardwareProfile == "" { + host.HardwareProfile = HardwareProfile + } + } + + if p.APIVIP == APIVIP { + // This name should resolve to exactly one address + vip, err := net.LookupHost("api." + c.ClusterDomain()) + if err != nil { + // This will fail validation and abort the install + p.APIVIP = fmt.Sprintf("DNS lookup failure: %s", err.Error()) + } else { + p.APIVIP = vip[0] + } + } + + if p.IngressVIP == IngressVIP { + // This name should resolve to exactly one address + vip, err := net.LookupHost("test.apps." + c.ClusterDomain()) + if err != nil { + // This will fail validation and abort the install + p.IngressVIP = fmt.Sprintf("DNS lookup failure: %s", err.Error()) + } else { + p.IngressVIP = vip[0] + } + } +} diff --git a/pkg/types/baremetal/doc.go b/pkg/types/baremetal/doc.go new file mode 100644 index 0000000000..db078844da --- /dev/null +++ b/pkg/types/baremetal/doc.go @@ -0,0 +1,6 @@ +// Package baremetal contains baremetal-specific structures for +// installer configuration and management. +package baremetal + +// Name is the name for the baremetal platform. +const Name string = "baremetal" diff --git a/pkg/types/baremetal/machinepool.go b/pkg/types/baremetal/machinepool.go new file mode 100644 index 0000000000..823736388c --- /dev/null +++ b/pkg/types/baremetal/machinepool.go @@ -0,0 +1,13 @@ +package baremetal + +// MachinePool stores the configuration for a machine pool installed +// on bare metal. +type MachinePool struct { +} + +// Set sets the values from `required` to `a`. +func (l *MachinePool) Set(required *MachinePool) { + if required == nil || l == nil { + return + } +} diff --git a/pkg/types/baremetal/metadata.go b/pkg/types/baremetal/metadata.go new file mode 100644 index 0000000000..bfa3b5ebc9 --- /dev/null +++ b/pkg/types/baremetal/metadata.go @@ -0,0 +1,7 @@ +package baremetal + +// Metadata contains baremetal metadata (e.g. for uninstalling the cluster). +type Metadata struct { + LibvirtURI string `json:"libvirtURI"` + IronicURI string `json:"ironicURI"` +} diff --git a/pkg/types/baremetal/platform.go b/pkg/types/baremetal/platform.go new file mode 100644 index 0000000000..55511933ea --- /dev/null +++ b/pkg/types/baremetal/platform.go @@ -0,0 +1,66 @@ +package baremetal + +// BMC stores the information about a baremetal host's management controller. +type BMC struct { + Username string `json:"username"` + Password string `json:"password"` + Address string `json:"address"` +} + +// Host stores all the configuration data for a baremetal host. +type Host struct { + Name string `json:"name,omitempty"` + BMC BMC `json:"bmc"` + Role string `json:"role"` + BootMACAddress string `json:"bootMACAddress"` + HardwareProfile string `json:"hardwareProfile"` +} + +// Image stores details about the locations of various images needed for deployment. +// FIXME: This should be determined by the installer once Ironic and image downloading occurs in bootstrap VM. +type Image struct { + Source string `json:"source"` + Checksum string `json:"checksum"` + DeployKernel string `json:"deployKernel"` + DeployRamdisk string `json:"deployRamdisk"` +} + +// Platform stores all the global configuration that all machinesets use. +type Platform struct { + // LibvirtURI is the identifier for the libvirtd connection. It must be + // reachable from the host where the installer is run. + // +optional + // Default is qemu:///system + LibvirtURI string `json:"libvirtURI,omitempty"` + + // IronicURI is the identifier for the Ironic connection. It must be + // reachable from the host where the installer is run. + // +optional + IronicURI string `json:"ironicURI,omitempty"` + + // External bridge is used for external communication. + // +optional + ExternalBridge string `json:"externalBridge,omitempty"` + + // Provisioning bridge is used for provisioning nodes. + // +optional + ProvisioningBridge string `json:"provisioningBridge,omitempty"` + + // Hosts is the information needed to create the objects in Ironic. + Hosts []*Host `json:"hosts"` + + // Images contains the information needed to provision a host + Image Image `json:"image"` + + // DefaultMachinePlatform is the default configuration used when + // installing on bare metal for machine pools which do not define their own + // platform configuration. + // +optional + DefaultMachinePlatform *MachinePool `json:"defaultMachinePlatform,omitempty"` + + // APIVIP is the VIP to use for internal API communication + APIVIP string `json:"apiVIP"` + + // IngressVIP is the VIP to use for ingress traffic + IngressVIP string `json:"ingressVIP"` +} diff --git a/pkg/types/baremetal/validation/machinepool.go b/pkg/types/baremetal/validation/machinepool.go new file mode 100644 index 0000000000..673d948276 --- /dev/null +++ b/pkg/types/baremetal/validation/machinepool.go @@ -0,0 +1,12 @@ +package validation + +import ( + "k8s.io/apimachinery/pkg/util/validation/field" + + "github.com/openshift/installer/pkg/types/baremetal" +) + +// ValidateMachinePool checks that the specified machine pool is valid. +func ValidateMachinePool(p *baremetal.MachinePool, fldPath *field.Path) field.ErrorList { + return field.ErrorList{} +} diff --git a/pkg/types/baremetal/validation/platform.go b/pkg/types/baremetal/validation/platform.go new file mode 100644 index 0000000000..da03d75f3e --- /dev/null +++ b/pkg/types/baremetal/validation/platform.go @@ -0,0 +1,44 @@ +package validation + +import ( + "github.com/openshift/installer/pkg/types/baremetal" + "github.com/openshift/installer/pkg/validate" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// ValidatePlatform checks that the specified platform is valid. +func ValidatePlatform(p *baremetal.Platform, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if err := validate.URI(p.LibvirtURI); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("libvirtURI"), p.LibvirtURI, err.Error())) + } + + if err := validate.URI(p.IronicURI); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ironicURI"), p.LibvirtURI, err.Error())) + } + + if err := validate.Interface(p.ExternalBridge); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("externalBridge"), p.ExternalBridge, err.Error())) + } + + if err := validate.Interface(p.ProvisioningBridge); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("provisioningBridge"), p.ProvisioningBridge, err.Error())) + } + + if p.Hosts == nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("hosts"), p.Hosts, "bare metal hosts are missing")) + } + + if p.DefaultMachinePlatform != nil { + allErrs = append(allErrs, ValidateMachinePool(p.DefaultMachinePlatform, fldPath.Child("defaultMachinePlatform"))...) + } + + if err := validate.IP(p.APIVIP); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("apiVIP"), p.APIVIP, err.Error())) + } + + if err := validate.IP(p.IngressVIP); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ingressVIP"), p.IngressVIP, err.Error())) + } + return allErrs +} diff --git a/pkg/types/clustermetadata.go b/pkg/types/clustermetadata.go index 49ec26fcd4..a398e96560 100644 --- a/pkg/types/clustermetadata.go +++ b/pkg/types/clustermetadata.go @@ -3,6 +3,7 @@ package types import ( "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/openstack" @@ -27,6 +28,7 @@ type ClusterPlatformMetadata struct { Libvirt *libvirt.Metadata `json:"libvirt,omitempty"` Azure *azure.Metadata `json:"azure,omitempty"` GCP *gcp.Metadata `json:"gcp,omitempty"` + BareMetal *baremetal.Metadata `json:"baremetal,omitempty"` } // Platform returns a string representation of the platform @@ -51,5 +53,8 @@ func (cpm *ClusterPlatformMetadata) Platform() string { if cpm.GCP != nil { return gcp.Name } + if cpm.BareMetal != nil { + return "baremetal" + } return "" } diff --git a/pkg/types/defaults/installconfig.go b/pkg/types/defaults/installconfig.go index 8ec411af15..879fec46ac 100644 --- a/pkg/types/defaults/installconfig.go +++ b/pkg/types/defaults/installconfig.go @@ -5,6 +5,7 @@ import ( "github.com/openshift/installer/pkg/types" awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults" azuredefaults "github.com/openshift/installer/pkg/types/azure/defaults" + baremetaldefaults "github.com/openshift/installer/pkg/types/baremetal/defaults" gcpdefaults "github.com/openshift/installer/pkg/types/gcp/defaults" libvirtdefaults "github.com/openshift/installer/pkg/types/libvirt/defaults" nonedefaults "github.com/openshift/installer/pkg/types/none/defaults" @@ -69,6 +70,8 @@ func SetInstallConfigDefaults(c *types.InstallConfig) { openstackdefaults.SetPlatformDefaults(c.Platform.OpenStack) case c.Platform.VSphere != nil: vspheredefaults.SetPlatformDefaults(c.Platform.VSphere, c) + case c.Platform.BareMetal != nil: + baremetaldefaults.SetPlatformDefaults(c.Platform.BareMetal, c) case c.Platform.None != nil: nonedefaults.SetPlatformDefaults(c.Platform.None) } diff --git a/pkg/types/installconfig.go b/pkg/types/installconfig.go index a272752370..b6c20e2c97 100644 --- a/pkg/types/installconfig.go +++ b/pkg/types/installconfig.go @@ -6,6 +6,7 @@ import ( "github.com/openshift/installer/pkg/ipnet" "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/none" @@ -34,6 +35,7 @@ var ( // hidden-but-supported platform names. This list isn't presented // to the user in the interactive wizard. HiddenPlatformNames = []string{ + baremetal.Name, none.Name, openstack.Name, vsphere.Name, @@ -99,6 +101,10 @@ type Platform struct { // +optional Azure *azure.Platform `json:"azure,omitempty"` + // BareMetal is the configuration used when installing on bare metal. + // +optional + BareMetal *baremetal.Platform `json:"baremetal,omitempty"` + // GCP is the configuration used when installing on Google Cloud Platform. // +optional GCP *gcp.Platform `json:"gcp,omitempty"` @@ -131,6 +137,8 @@ func (p *Platform) Name() string { return aws.Name case p.Azure != nil: return azure.Name + case p.BareMetal != nil: + return baremetal.Name case p.GCP != nil: return gcp.Name case p.Libvirt != nil: diff --git a/pkg/types/machinepools.go b/pkg/types/machinepools.go index 4aa1bbe998..99b42b6957 100644 --- a/pkg/types/machinepools.go +++ b/pkg/types/machinepools.go @@ -3,6 +3,7 @@ package types import ( "github.com/openshift/installer/pkg/types/aws" "github.com/openshift/installer/pkg/types/azure" + "github.com/openshift/installer/pkg/types/baremetal" "github.com/openshift/installer/pkg/types/gcp" "github.com/openshift/installer/pkg/types/libvirt" "github.com/openshift/installer/pkg/types/openstack" @@ -48,6 +49,9 @@ type MachinePoolPlatform struct { // Azure is the configuration used when installing on OpenStack. Azure *azure.MachinePool `json:"azure,omitempty"` + // BareMetal is the configuration used when installing on bare metal. + BareMetal *baremetal.MachinePool `json:"baremetal,omitempty"` + // GCP is the configuration used when installing on GCP GCP *gcp.MachinePool `json:"gcp,omitempty"` @@ -72,6 +76,8 @@ func (p *MachinePoolPlatform) Name() string { return aws.Name case p.Azure != nil: return azure.Name + case p.BareMetal != nil: + return baremetal.Name case p.GCP != nil: return gcp.Name case p.Libvirt != nil: diff --git a/pkg/types/validation/installconfig.go b/pkg/types/validation/installconfig.go index 85007d769b..6f399100f0 100644 --- a/pkg/types/validation/installconfig.go +++ b/pkg/types/validation/installconfig.go @@ -17,6 +17,8 @@ import ( awsvalidation "github.com/openshift/installer/pkg/types/aws/validation" "github.com/openshift/installer/pkg/types/azure" azurevalidation "github.com/openshift/installer/pkg/types/azure/validation" + "github.com/openshift/installer/pkg/types/baremetal" + baremetalvalidation "github.com/openshift/installer/pkg/types/baremetal/validation" "github.com/openshift/installer/pkg/types/gcp" gcpvalidation "github.com/openshift/installer/pkg/types/gcp/validation" "github.com/openshift/installer/pkg/types/libvirt" @@ -240,6 +242,11 @@ func validatePlatform(platform *types.Platform, fldPath *field.Path, openStackVa if platform.VSphere != nil { validate(vsphere.Name, platform.VSphere, func(f *field.Path) field.ErrorList { return vspherevalidation.ValidatePlatform(platform.VSphere, f) }) } + if platform.BareMetal != nil { + validate(baremetal.Name, platform.BareMetal, func(f *field.Path) field.ErrorList { + return baremetalvalidation.ValidatePlatform(platform.BareMetal, f) + }) + } return allErrs } diff --git a/pkg/types/validation/installconfig_test.go b/pkg/types/validation/installconfig_test.go index ea68e4819b..9c44ff725b 100644 --- a/pkg/types/validation/installconfig_test.go +++ b/pkg/types/validation/installconfig_test.go @@ -361,7 +361,7 @@ func TestValidateInstallConfig(t *testing.T) { c.Platform = types.Platform{} return c }(), - expectedError: `^platform: Invalid value: "": must specify one of the platforms \(aws, azure, gcp, none, openstack, vsphere\)$`, + expectedError: `^platform: Invalid value: "": must specify one of the platforms \(aws, azure, baremetal, gcp, none, openstack, vsphere\)$`, }, { name: "multiple platforms", @@ -392,7 +392,7 @@ func TestValidateInstallConfig(t *testing.T) { } return c }(), - expectedError: `^platform: Invalid value: "libvirt": must specify one of the platforms \(aws, azure, gcp, none, openstack, vsphere\)$`, + expectedError: `^platform: Invalid value: "libvirt": must specify one of the platforms \(aws, azure, baremetal, gcp, none, openstack, vsphere\)$`, }, { name: "invalid libvirt platform", @@ -404,7 +404,7 @@ func TestValidateInstallConfig(t *testing.T) { c.Platform.Libvirt.URI = "" return c }(), - expectedError: `^\[platform: Invalid value: "libvirt": must specify one of the platforms \(aws, azure, gcp, none, openstack, vsphere\), platform\.libvirt\.uri: Invalid value: "": invalid URI "" \(no scheme\)]$`, + expectedError: `^\[platform: Invalid value: "libvirt": must specify one of the platforms \(aws, azure, baremetal, gcp, none, openstack, vsphere\), platform\.libvirt\.uri: Invalid value: "": invalid URI "" \(no scheme\)]$`, }, { name: "valid none platform", diff --git a/pkg/types/validation/machinepools.go b/pkg/types/validation/machinepools.go index 52064f2195..33867a09c1 100644 --- a/pkg/types/validation/machinepools.go +++ b/pkg/types/validation/machinepools.go @@ -10,6 +10,8 @@ import ( awsvalidation "github.com/openshift/installer/pkg/types/aws/validation" "github.com/openshift/installer/pkg/types/azure" azurevalidation "github.com/openshift/installer/pkg/types/azure/validation" + "github.com/openshift/installer/pkg/types/baremetal" + baremetalvalidation "github.com/openshift/installer/pkg/types/baremetal/validation" "github.com/openshift/installer/pkg/types/libvirt" libvirtvalidation "github.com/openshift/installer/pkg/types/libvirt/validation" "github.com/openshift/installer/pkg/types/openstack" @@ -71,5 +73,8 @@ func validateMachinePoolPlatform(platform *types.Platform, p *types.MachinePoolP if p.OpenStack != nil { validate(openstack.Name, p.OpenStack, func(f *field.Path) field.ErrorList { return openstackvalidation.ValidateMachinePool(p.OpenStack, f) }) } + if p.BareMetal != nil { + validate(baremetal.Name, p.BareMetal, func(f *field.Path) field.ErrorList { return baremetalvalidation.ValidateMachinePool(p.BareMetal, f) }) + } return allErrs } diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go index 1c1877bed9..03d51b6e69 100644 --- a/pkg/validate/validate.go +++ b/pkg/validate/validate.go @@ -132,3 +132,26 @@ func URIWithProtocol(uri string, protocol string) error { } return nil } + +// IP validates if a string is a valid IP. +func IP(ip string) error { + addr := net.ParseIP(ip) + if addr == nil { + return fmt.Errorf("'%s' is not a valid IP", ip) + } + return nil +} + +// Interface validates if a string is a valid network interface +func Interface(iface string) error { + if _, err := net.InterfaceByName(iface); err != nil { + return fmt.Errorf("%s is not a valid network interface: %s", iface, err) + } + return nil +} + +// MAC validates that a value is a valid mac address +func MAC(addr string) error { + _, err := net.ParseMAC(addr) + return err +}