pkg/asset: add ClusterK8sIO, machines.Worker assets

Add assets that can create the machinesets for workers.
This depends on the cluster-apiserver to be deployed by the machine-api-operator,
so the tectonic.sh will block on creating these machinesets objects until then.

Using specific AMI for worker machinesets to keep consistency with
current MAO implementation.

pkg/asset/machines/aws/aws.go

@@ -0,0 +1,76 @@
+// Package aws generates Machine objects for aws.
+package aws
+
+import (
+	"text/template"
+
+	"github.com/openshift/installer/pkg/types"
+)
+
+// Config is used to generate the machine.
+type Config struct {
+	ClusterName string
+	Replicas    int64
+	AMIID       string
+	Tags        map[string]string
+	Region      string
+	Machine     types.AWSMachinePoolPlatform
+}
+
+// WorkerMachineSetTmpl is template for worker machineset.
+var WorkerMachineSetTmpl = template.Must(template.New("aws-worker-machineset").Parse(`
+apiVersion: cluster.k8s.io/v1alpha1
+kind: MachineSet
+metadata:
+  name: {{.ClusterName}}-worker-0
+  namespace: openshift-cluster-api
+  labels:
+    sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+    sigs.k8s.io/cluster-api-machine-role: worker
+    sigs.k8s.io/cluster-api-machine-type: worker
+spec:
+  replicas: {{.Replicas}}
+  selector:
+    matchLabels:
+      sigs.k8s.io/cluster-api-machineset: worker
+      sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+  template:
+    metadata:
+      labels:
+        sigs.k8s.io/cluster-api-machineset: worker
+        sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+        sigs.k8s.io/cluster-api-machine-role: worker
+        sigs.k8s.io/cluster-api-machine-type: worker
+    spec:
+      providerConfig:
+        value:
+          apiVersion: aws.cluster.k8s.io/v1alpha1
+          kind: AWSMachineProviderConfig
+          ami:
+            id: {{.AMIID}}
+          instanceType: {{.Machine.InstanceType}}
+          placement:
+            region: {{.Region}}
+          subnet:
+            filters:
+            - name: "tag:Name"
+              values:
+              - "{{.ClusterName}}-worker-*"
+          iamInstanceProfile:
+            id: "{{.ClusterName}}-worker-profile"
+          tags:
+{{- range $key,$value := .Tags}}
+            - name: "{{$key}}"
+              value: "{{$value}}"
+{{- end}}
+          securityGroups:
+            - filters:
+              - name: "tag:Name"
+                values:
+                - "{{.ClusterName}}_worker_sg"
+          userDataSecret:
+            name: worker-user-data
+      versions:
+        kubelet: ""
+        controlPlane: ""
+`))

pkg/asset/machines/cluster_k8s_io.go

@@ -0,0 +1,73 @@
+package machines
+
+import (
+	"bytes"
+	"text/template"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/types"
+)
+
+// ClusterK8sIO generates the `Cluster.cluster.k8s.io/v1alpha1` object.
+type ClusterK8sIO struct {
+	Raw []byte
+}
+
+var _ asset.Asset = (*ClusterK8sIO)(nil)
+
+// Name returns a human friendly name for the ClusterK8sIO Asset.
+func (c *ClusterK8sIO) Name() string {
+	return "Cluster.cluster.k8s.io/v1alpha1"
+}
+
+// Dependencies returns all of the dependencies directly needed by the
+// ClusterK8sIO asset
+func (c *ClusterK8sIO) Dependencies() []asset.Asset {
+	return []asset.Asset{
+		&installconfig.InstallConfig{},
+	}
+}
+
+// Generate generates the Worker asset.
+func (c *ClusterK8sIO) Generate(dependencies asset.Parents) error {
+	installconfig := &installconfig.InstallConfig{}
+	dependencies.Get(installconfig)
+
+	c.Raw = clusterK8sIO(installconfig.Config)
+	return nil
+}
+
+var clusterK8sIOTmpl = template.Must(template.New("cluster").Parse(`
+apiVersion: "cluster.k8s.io/v1alpha1"
+kind: Cluster
+metadata:
+  name: {{.Name}}
+  namespace: openshift-cluster-api
+spec:
+  clusterNetwork:
+    services:
+      cidrBlocks:
+      - {{.ServiceCIDR}}
+    pods:
+      cidrBlocks:
+      - {{.PodCIDR}}
+    serviceDomain: unused
+`))
+
+func clusterK8sIO(ic *types.InstallConfig) []byte {
+	templateData := struct {
+		Name        string
+		ServiceCIDR string
+		PodCIDR     string
+	}{
+		Name:        ic.ObjectMeta.Name,
+		ServiceCIDR: ic.Networking.ServiceCIDR.String(),
+		PodCIDR:     ic.Networking.PodCIDR.String(),
+	}
+	buf := &bytes.Buffer{}
+	if err := clusterK8sIOTmpl.Execute(buf, templateData); err != nil {
+		panic(err)
+	}
+	return buf.Bytes()
+}

pkg/asset/machines/doc.go

@@ -0,0 +1,2 @@
+// Package machines is responsible for creating Machine objects for machinepools.
+package machines

pkg/asset/machines/libvirt/libvirt.go

@@ -0,0 +1,61 @@
+// Package libvirt generates Machine objects for libvirt.
+package libvirt
+
+import (
+	"text/template"
+
+	"github.com/openshift/installer/pkg/types"
+)
+
+// Config is used to generate the machine.
+type Config struct {
+	ClusterName string
+	Replicas    int64
+	Platform    types.LibvirtPlatform
+}
+
+// WorkerMachineSetTmpl is template for worker machineset.
+var WorkerMachineSetTmpl = template.Must(template.New("worker-machineset").Parse(`
+apiVersion: cluster.k8s.io/v1alpha1
+kind: MachineSet
+metadata:
+  name: {{.ClusterName}}-worker-0
+  namespace: openshift-cluster-api
+  labels:
+    sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+    sigs.k8s.io/cluster-api-machine-role: worker
+    sigs.k8s.io/cluster-api-machine-type: worker
+spec:
+  replicas: {{.Replicas}}
+  selector:
+    matchLabels:
+      sigs.k8s.io/cluster-api-machineset: worker
+      sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+      sigs.k8s.io/cluster-api-machine-role: worker
+      sigs.k8s.io/cluster-api-machine-type: worker
+  template:
+    metadata:
+      labels:
+        sigs.k8s.io/cluster-api-machineset: worker
+        sigs.k8s.io/cluster-api-cluster: {{.ClusterName}}
+        sigs.k8s.io/cluster-api-machine-role: worker
+        sigs.k8s.io/cluster-api-machine-type: worker
+    spec:
+      providerConfig:
+        value:
+          apiVersion: libvirtproviderconfig/v1alpha1
+          kind: LibvirtMachineProviderConfig
+          domainMemory: 2048
+          domainVcpu: 2
+          ignKey: /var/lib/libvirt/images/worker.ign
+          volume:
+            poolName: default
+            baseVolumeID: /var/lib/libvirt/images/coreos_base
+          networkInterfaceName: {{.Platform.Network.Name}}
+          networkInterfaceAddress: {{.Platform.Network.IPRange}}
+          autostart: false
+          uri: {{.Platform.URI}}
+      versions:
+        kubelet: ""
+        controlPlane: ""
+`))

pkg/asset/machines/userdata.go

@@ -0,0 +1,35 @@
+package machines
+
+import (
+	"bytes"
+	"encoding/base64"
+	"text/template"
+
+	"github.com/pkg/errors"
+)
+
+var userDataTmpl = template.Must(template.New("user-data").Parse(`
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{.Name}}
+  namespace: openshift-cluster-api
+type: Opaque
+data:
+  userData: {{.UserDataContent}}
+`))
+
+func userData(secretName string, content []byte) ([]byte, error) {
+	templateData := struct {
+		Name            string
+		UserDataContent string
+	}{
+		Name:            secretName,
+		UserDataContent: base64.StdEncoding.EncodeToString(content),
+	}
+	buf := &bytes.Buffer{}
+	if err := userDataTmpl.Execute(buf, templateData); err != nil {
+		return nil, errors.Wrap(err, "failed to execute content.UserDataTmpl")
+	}
+	return buf.Bytes(), nil
+}

pkg/asset/machines/worker.go

@@ -0,0 +1,127 @@
+package machines
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"text/template"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/asset/ignition/machine"
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/machines/aws"
+	"github.com/openshift/installer/pkg/asset/machines/libvirt"
+	"github.com/openshift/installer/pkg/rhcos"
+	"github.com/openshift/installer/pkg/types"
+)
+
+func defaultAWSMachinePoolPlatform() types.AWSMachinePoolPlatform {
+	return types.AWSMachinePoolPlatform{
+		InstanceType: "t2.medium",
+	}
+}
+
+// Worker generates the machinesets for `worker` machine pool.
+type Worker struct {
+	MachineSetRaw     []byte
+	UserDataSecretRaw []byte
+}
+
+var _ asset.Asset = (*Worker)(nil)
+
+// Name returns a human friendly name for the Worker Asset.
+func (w *Worker) Name() string {
+	return "Worker Machines"
+}
+
+// Dependencies returns all of the dependencies directly needed by the
+// Worker asset
+func (w *Worker) Dependencies() []asset.Asset {
+	return []asset.Asset{
+		&installconfig.InstallConfig{},
+		&machine.Worker{},
+	}
+}
+
+// Generate generates the Worker asset.
+func (w *Worker) Generate(dependencies asset.Parents) error {
+	installconfig := &installconfig.InstallConfig{}
+	wign := &machine.Worker{}
+	dependencies.Get(installconfig, wign)
+
+	var err error
+	w.UserDataSecretRaw, err = userData("worker-user-data", wign.File.Data)
+	if err != nil {
+		return errors.Wrap(err, "failed to create user-data secret for worker machines")
+	}
+
+	ic := installconfig.Config
+	pool := workerPool(ic.Machines)
+	numOfWorkers := int64(0)
+	if pool.Replicas != nil {
+		numOfWorkers = *pool.Replicas
+	}
+
+	switch ic.Platform.Name() {
+	case "aws":
+		config := aws.Config{
+			ClusterName: ic.ObjectMeta.Name,
+			Replicas:    numOfWorkers,
+			Region:      ic.Platform.AWS.Region,
+			Machine:     defaultAWSMachinePoolPlatform(),
+		}
+
+		tags := map[string]string{
+			"tectonicClusterID": ic.ClusterID,
+			// Info: https://github.com/openshift/cluster-api-provider-aws/issues/73
+			// fmt.Sprintf("kubernetes.io/cluster/%s", ic.ObjectMeta.Name): "owned",
+		}
+		for k, v := range ic.Platform.AWS.UserTags {
+			tags[k] = v
+		}
+		config.Tags = tags
+
+		config.Machine.Set(ic.Platform.AWS.DefaultMachinePlatform)
+		config.Machine.Set(pool.Platform.AWS)
+
+		ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
+		defer cancel()
+		ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, config.Region)
+		if err != nil {
+			return errors.Wrap(err, "failed to determine default AMI")
+		}
+		config.AMIID = ami
+
+		w.MachineSetRaw = applyTemplateData(aws.WorkerMachineSetTmpl, config)
+	case "libvirt":
+		config := libvirt.Config{
+			ClusterName: ic.ObjectMeta.Name,
+			Replicas:    numOfWorkers,
+			Platform:    *ic.Platform.Libvirt,
+		}
+		w.MachineSetRaw = applyTemplateData(libvirt.WorkerMachineSetTmpl, config)
+	default:
+		return fmt.Errorf("invalid Platform")
+	}
+	return nil
+}
+
+func workerPool(pools []types.MachinePool) types.MachinePool {
+	for idx, pool := range pools {
+		if pool.Name == "worker" {
+			return pools[idx]
+		}
+	}
+	return types.MachinePool{}
+}
+
+func applyTemplateData(template *template.Template, templateData interface{}) []byte {
+	buf := &bytes.Buffer{}
+	if err := template.Execute(buf, templateData); err != nil {
+		panic(err)
+	}
+	return buf.Bytes()
+}

pkg/asset/manifests/machine-api-operator.go

@@ -108,7 +108,7 @@ func (mao *machineAPIOperator) Generate(dependencies asset.Parents) error {
 			Region:           installConfig.Config.Platform.AWS.Region,
 			AvailabilityZone: "",
 			Image:            ami,
-			Replicas:         int(*installConfig.Config.Machines[1].Replicas),
+			Replicas:         0, // setting replicas to 0 so that MAO doesn't create competing MachineSets
 		}
 	case installConfig.Config.Platform.Libvirt != nil:
 		mao.Config.Libvirt = &libvirtConfig{
@@ -116,14 +116,14 @@ func (mao *machineAPIOperator) Generate(dependencies asset.Parents) error {
 			URI:         installConfig.Config.Platform.Libvirt.URI,
 			NetworkName: installConfig.Config.Platform.Libvirt.Network.Name,
 			IPRange:     installConfig.Config.Platform.Libvirt.Network.IPRange,
-			Replicas:    int(*installConfig.Config.Machines[1].Replicas),
+			Replicas:    0, // setting replicas to 0 so that MAO doesn't create competing MachineSets
 		}
 	case installConfig.Config.Platform.OpenStack != nil:
 		mao.Config.OpenStack = &openstackConfig{
 			ClusterName: installConfig.Config.ObjectMeta.Name,
 			ClusterID:   installConfig.Config.ClusterID,
 			Region:      installConfig.Config.Platform.OpenStack.Region,
-			Replicas:    int(*installConfig.Config.Machines[1].Replicas),
+			Replicas:    0, // setting replicas to 0 so that MAO doesn't create competing MachineSets
 		}
 	default:
 		return errors.Errorf("unknown provider for machine-api-operator")

pkg/asset/manifests/tectonic.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	"github.com/openshift/installer/pkg/asset/machines"
 	content "github.com/openshift/installer/pkg/asset/manifests/content/tectonic"
 	"github.com/openshift/installer/pkg/asset/tls"
 )
@@ -30,6 +31,8 @@ func (t *Tectonic) Dependencies() []asset.Asset {
 		&installconfig.InstallConfig{},
 		&tls.IngressCertKey{},
 		&tls.KubeCA{},
+		&machines.ClusterK8sIO{},
+		&machines.Worker{},
 	}
 }
 
@@ -38,7 +41,9 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error {
 	installConfig := &installconfig.InstallConfig{}
 	ingressCertKey := &tls.IngressCertKey{}
 	kubeCA := &tls.KubeCA{}
-	dependencies.Get(installConfig, ingressCertKey, kubeCA)
+	clusterk8sio := &machines.ClusterK8sIO{}
+	worker := &machines.Worker{}
+	dependencies.Get(installConfig, ingressCertKey, kubeCA, clusterk8sio, worker)
 
 	templateData := &tectonicTemplateData{
 		IngressCaCert:                          base64.StdEncoding.EncodeToString(kubeCA.Cert()),
@@ -55,22 +60,25 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error {
 	}
 
 	assetData := map[string][]byte{
-		"99_binding-discovery.yaml":                  []byte(content.BindingDiscovery),
-		"99_kube-addon-00-appversion.yaml":           []byte(content.AppVersionKubeAddon),
-		"99_kube-addon-01-operator.yaml":             applyTemplateData(content.KubeAddonOperator, templateData),
-		"99_kube-core-00-appversion.yaml":            []byte(content.AppVersionKubeCore),
-		"99_kube-core-00-operator.yaml":              applyTemplateData(content.KubeCoreOperator, templateData),
-		"99_role-admin.yaml":                         []byte(content.RoleAdmin),
-		"99_role-user.yaml":                          []byte(content.RoleUser),
-		"99_tectonic-ingress-00-appversion.yaml":     []byte(content.AppVersionTectonicIngress),
-		"99_tectonic-ingress-01-cluster-config.yaml": applyTemplateData(content.ClusterConfigTectonicIngress, templateData),
-		"99_tectonic-ingress-02-tls.yaml":            applyTemplateData(content.TLSTectonicIngress, templateData),
-		"99_tectonic-ingress-03-pull.json":           applyTemplateData(content.PullTectonicIngress, templateData),
-		"99_tectonic-ingress-04-svc-account.yaml":    []byte(content.SvcAccountTectonicIngress),
-		"99_tectonic-ingress-05-operator.yaml":       applyTemplateData(content.TectonicIngressControllerOperator, templateData),
-		"99_tectonic-system-00-binding-admin.yaml":   []byte(content.BindingAdmin),
-		"99_tectonic-system-01-ca-cert.yaml":         applyTemplateData(content.CaCertTectonicSystem, templateData),
-		"99_tectonic-system-02-pull.json":            applyTemplateData(content.PullTectonicSystem, templateData),
+		"99_binding-discovery.yaml":                             []byte(content.BindingDiscovery),
+		"99_kube-addon-00-appversion.yaml":                      []byte(content.AppVersionKubeAddon),
+		"99_kube-addon-01-operator.yaml":                        applyTemplateData(content.KubeAddonOperator, templateData),
+		"99_kube-core-00-appversion.yaml":                       []byte(content.AppVersionKubeCore),
+		"99_kube-core-00-operator.yaml":                         applyTemplateData(content.KubeCoreOperator, templateData),
+		"99_openshift-cluster-api_cluster.yaml":                 clusterk8sio.Raw,
+		"99_openshift-cluster-api_worker-machineset.yaml":       worker.MachineSetRaw,
+		"99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw,
+		"99_role-admin.yaml":                                    []byte(content.RoleAdmin),
+		"99_role-user.yaml":                                     []byte(content.RoleUser),
+		"99_tectonic-ingress-00-appversion.yaml":                []byte(content.AppVersionTectonicIngress),
+		"99_tectonic-ingress-01-cluster-config.yaml":            applyTemplateData(content.ClusterConfigTectonicIngress, templateData),
+		"99_tectonic-ingress-02-tls.yaml":                       applyTemplateData(content.TLSTectonicIngress, templateData),
+		"99_tectonic-ingress-03-pull.json":                      applyTemplateData(content.PullTectonicIngress, templateData),
+		"99_tectonic-ingress-04-svc-account.yaml":               []byte(content.SvcAccountTectonicIngress),
+		"99_tectonic-ingress-05-operator.yaml":                  applyTemplateData(content.TectonicIngressControllerOperator, templateData),
+		"99_tectonic-system-00-binding-admin.yaml":              []byte(content.BindingAdmin),
+		"99_tectonic-system-01-ca-cert.yaml":                    applyTemplateData(content.CaCertTectonicSystem, templateData),
+		"99_tectonic-system-02-pull.json":                       applyTemplateData(content.PullTectonicSystem, templateData),
 	}
 
 	t.FileList = make([]*asset.File, 0, len(assetData))