openstack/nova: replace cloud-init with ignition

openstack-novanet/config.tf

@@ -1,11 +1,11 @@
 variable "flavor_id" {
   type    = "string"
-  default = "5cf64088-893b-46b5-9bb1-ee020277635d"
+  default = "bbcb7eb5-5c8d-498f-9d7e-307c575d3566"
 }
 
 variable "image_id" {
   type    = "string"
-  default = "3a0c0bac-fa91-4c96-bfcb-ee215ba1cd4d"
+  default = "acdcd535-5408-40f3-8e88-ad8ebb6507e6"
 }
 
 variable "tectonic_version" {
@@ -13,7 +13,7 @@ variable "tectonic_version" {
   default = "v1.5.2_coreos.1"
 }
 
-variable "controller_count" {
+variable "master_count" {
   type    = "string"
   default = "1"
 }

openstack-novanet/dns.tf

@@ -7,7 +7,7 @@ resource "aws_route53_record" "tectonic-api" {
   name    = "${var.cluster_name}-k8s"
   type    = "A"
   ttl     = "60"
-  records = ["${openstack_compute_instance_v2.control_node.*.access_ip_v4}"]
+  records = ["${openstack_compute_instance_v2.master_node.*.access_ip_v4}"]
 }
 
 resource "aws_route53_record" "tectonic-console" {
@@ -26,13 +26,13 @@ resource "aws_route53_record" "etcd" {
   records = ["${openstack_compute_instance_v2.etcd_node.*.access_ip_v4}"]
 }
 
-resource "aws_route53_record" "controller_nodes" {
-  count   = "${var.controller_count}"
+resource "aws_route53_record" "master_nodes" {
+  count   = "${var.master_count}"
   zone_id = "${data.aws_route53_zone.tectonic.zone_id}"
-  name    = "${var.cluster_name}-controller-${count.index}"
+  name    = "${var.cluster_name}-master-${count.index}"
   type    = "A"
   ttl     = "60"
-  records = ["${openstack_compute_instance_v2.control_node.*.access_ip_v4[count.index]}"]
+  records = ["${openstack_compute_instance_v2.master_node.*.access_ip_v4[count.index]}"]
 }
 
 resource "aws_route53_record" "worker_nodes" {

openstack-novanet/etcd.tf

@@ -10,7 +10,7 @@ resource "openstack_compute_instance_v2" "etcd_node" {
     role = "etcd"
   }
 
-  user_data    = "${file("${path.module}/userdata-etcd.yml")}"
+  user_data    = "${ignition_config.etcd.*.rendered[count.index]}"
   config_drive = false
 }
 

openstack-novanet/ignition-etcd.tf

@@ -0,0 +1,44 @@
+resource "ignition_systemd_unit" "etcd2" {
+  name = "etcd2.service"
+  enable = false
+}
+
+resource "ignition_systemd_unit" "etcd" {
+  name = "etcd.service"
+  enable = false
+}
+
+resource "ignition_systemd_unit" "etcd_member" {
+  name = "etcd-member.service"
+
+  dropin {
+    name = "40-etcd-cluster.conf"
+    content = <<EOF
+[Service]
+Environment="ETCD_IMAGE_TAG=v3.1.0"
+ExecStartPre=/usr/bin/sh -c '/usr/bin/systemctl set-environment COREOS_PRIVATE_IPV4=$$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)'
+ExecStart=
+ExecStart=/usr/lib/coreos/etcd-wrapper \
+--name=etcd \
+--advertise-client-urls=http://$${COREOS_PRIVATE_IPV4}:2379 \
+--initial-advertise-peer-urls=http://$${COREOS_PRIVATE_IPV4}:2380 \
+--listen-client-urls=http://0.0.0.0:2379 \
+--listen-peer-urls=http://0.0.0.0:2380 \
+--initial-cluster=etcd=http://$${COREOS_PRIVATE_IPV4}:2380
+EOF
+  }
+}
+
+resource "ignition_config" "etcd" {
+  count    = "${var.etcd_count}"
+
+  users = [
+    "${ignition_user.core.id}",
+  ]
+
+  systemd = [
+    "${ignition_systemd_unit.etcd2.id}",
+    "${ignition_systemd_unit.etcd.id}",
+    "${ignition_systemd_unit.etcd_member.id}",
+  ]
+}

openstack-novanet/ignition-master.tf

@@ -0,0 +1,220 @@
+resource "ignition_file" "master_bootkube_dir" {
+  path       = "/opt/bootkube/.empty"
+  mode       = 0420
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = ""
+  }
+}
+
+resource "ignition_file" "master_kubelet_env" {
+  path       = "/etc/kubernetes/kubelet.env"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "KUBELET_IMAGE_URL=quay.io/coreos/hyperkube KUBELET_IMAGE_TAG=${var.tectonic_version}"
+  }
+}
+
+resource "ignition_file" "master_kubeconfig" {
+  path       = "/etc/kubernetes/kubeconfig"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/auth/kubeconfig")}"
+  }
+}
+
+resource "ignition_file" "master_max_user_watches_conf" {
+  path       = "/etc/sysctl.d/max-user-watches.conf"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "fs.inotify.max_user_watches=16184"
+  }
+}
+
+resource "ignition_file" "master_ca_pem" {
+  path       = "/etc/kubernetes/ssl/ca.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/ca.crt")}"
+  }
+}
+
+resource "ignition_file" "master_client_pem" {
+  path       = "/etc/kubernetes/ssl/client.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/kubelet.crt")}"
+  }
+}
+
+resource "ignition_file" "master_client_key" {
+  path       = "/etc/kubernetes/ssl/client.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/kubelet.key")}"
+  }
+}
+
+resource "ignition_file" "master_resolv_conf" {
+  path       = "/etc/resolv.conf"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = <<EOF
+search ${var.base_domain}
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+EOF
+  }
+}
+
+resource "ignition_file" "master_hostname" {
+  count      = "${var.master_count}"
+  path       = "/etc/hostname"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${var.cluster_name}-master-${count.index}"
+  }
+}
+
+resource "ignition_systemd_unit" "master_locksmithd" {
+  name = "locksmithd.service"
+  enable = false
+
+  dropin {
+    name = "40-etcd-lock.conf"
+    content = <<EOF
+[Service]
+Environment="REBOOT_STRATEGY=off"
+Environment="LOCKSMITHCTL_ENDPOINT=http://localhost:2379"
+EOF
+  }
+}
+
+resource "ignition_systemd_unit" "master_etcd-member" {
+  name = "etcd-member.service"
+
+  dropin {
+    name = "40-etcd-gateway.conf"
+    content = <<EOF
+[Service]
+Type=simple
+Environment="ETCD_IMAGE_TAG=v3.1.0"
+ExecStart=
+ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \
+      --listen-addr=127.0.0.1:2379 \
+      --endpoints=${aws_route53_record.etcd.fqdn}:2379
+EOF
+  }
+}
+
+resource "ignition_systemd_unit" "master_bootkube" {
+  name = "bootkube.service"
+  enable = false
+  content = <<EOF
+[Unit]
+Description=Bootstrap a Kubernetes control plane with a temp api-server
+
+[Service]
+Type=oneshot
+WorkingDirectory=/opt/bootkube
+ExecStartPre=-chmod a+x /opt/bootkube/assets/bootkube-start
+ExecStart=/opt/bootkube/assets/bootkube-start
+EOF
+}
+
+resource "ignition_systemd_unit" "master_kubelet" {
+  name = "kubelet.service"
+  enable = true
+  content = <<EOF
+[Unit]
+Description=Kubelet via Hyperkube ACI
+
+[Service]
+Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
+  --volume=resolv,kind=host,source=/etc/resolv.conf \
+  --mount volume=resolv,target=/etc/resolv.conf \
+  --volume var-lib-cni,kind=host,source=/var/lib/cni \
+  --mount volume=var-lib-cni,target=/var/lib/cni \
+  --volume var-log,kind=host,source=/var/log \
+  --mount volume=var-log,target=/var/log"
+Environment="KUBELET_IMAGE_URL=quay.io/coreos/hyperkube" "KUBELET_IMAGE_TAG=${var.tectonic_version}"
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+ExecStartPre=/bin/mkdir -p /var/lib/cni
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
+ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
+  --cni-conf-dir=/etc/kubernetes/cni/net.d \
+  --network-plugin=cni \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --exit-on-lock-contention \
+  --pod-manifest-path=/etc/kubernetes/manifests \
+  --allow-privileged=true \
+  --node-labels=master=true \
+  --minimum-container-ttl-duration=6m0s \
+  --cluster_dns=10.3.0.10 \
+  --cluster_domain=cluster.local
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+EOF
+}
+
+resource "ignition_config" "master" {
+  count    = "${var.master_count}"
+
+  users = [
+    "${ignition_user.core.id}",
+  ]
+
+  files = [
+    "${ignition_file.master_bootkube_dir.id}",
+    "${ignition_file.master_kubelet_env.id}",
+    "${ignition_file.master_kubeconfig.id}",
+    "${ignition_file.master_max_user_watches_conf.id}",
+    "${ignition_file.master_ca_pem.id}",
+    "${ignition_file.master_client_pem.id}",
+    "${ignition_file.master_client_key.id}",
+    "${ignition_file.master_resolv_conf.id}",
+    "${ignition_file.master_hostname.*.id[count.index]}",
+  ]
+
+  systemd = [
+    "${ignition_systemd_unit.master_locksmithd.id}",
+    "${ignition_systemd_unit.master_etcd-member.id}",
+    "${ignition_systemd_unit.master_bootkube.id}",
+    "${ignition_systemd_unit.master_kubelet.id}",
+  ]
+}

openstack-novanet/ignition-worker.tf

@@ -0,0 +1,167 @@
+resource "ignition_file" "worker_hostname" {
+  count      = "${var.worker_count}"
+  path       = "/etc/hostname"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${var.cluster_name}-worker-${count.index}"
+  }
+}
+
+resource "ignition_file" "worker_kubeconfig" {
+  path       = "/etc/kubernetes/kubeconfig"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/auth/kubeconfig")}"
+  }
+}
+
+resource "ignition_file" "worker_ca_pem" {
+  path       = "/etc/kubernetes/ssl/ca.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/ca.crt")}"
+  }
+}
+
+resource "ignition_file" "worker_client_pem" {
+  path       = "/etc/kubernetes/ssl/client.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/kubelet.crt")}"
+  }
+}
+
+resource "ignition_file" "worker_client_key" {
+  path       = "/etc/kubernetes/ssl/client.pem"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = "${file("${path.root}/../assets/tls/kubelet.key")}"
+  }
+}
+
+resource "ignition_file" "worker_resolv_conf" {
+  path       = "/etc/resolv.conf"
+  mode       = 0644
+  uid        = 0
+  filesystem = "root"
+
+  content {
+    content = <<EOF
+search ${var.base_domain}
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+EOF
+  }
+}
+
+resource "ignition_systemd_unit" "worker_locksmithd" {
+  name = "locksmithd.service"
+  enable = false
+
+  dropin {
+    name = "40-etcd-lock.conf"
+    content = <<EOF
+[Service]
+Environment="REBOOT_STRATEGY=off"
+Environment="LOCKSMITHCTL_ENDPOINT=http://localhost:2379"
+EOF
+  }
+}
+
+resource "ignition_systemd_unit" "worker_etcd-member" {
+  name = "etcd-member.service"
+
+  dropin {
+    name = "40-etcd-gateway.conf"
+    content = <<EOF
+[Service]
+Type=simple
+Environment="ETCD_IMAGE_TAG=v3.1.0"
+ExecStart=
+ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \
+      --listen-addr=127.0.0.1:2379 \
+      --endpoints=${aws_route53_record.etcd.fqdn}:2379
+EOF
+  }
+}
+
+resource "ignition_systemd_unit" "worker_kubelet" {
+  name = "kubelet.service"
+  enable = true
+  content = <<EOF
+[Unit]
+Description=Kubelet via Hyperkube ACI
+
+[Service]
+Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
+  --volume=resolv,kind=host,source=/etc/resolv.conf \
+  --mount volume=resolv,target=/etc/resolv.conf \
+  --volume var-lib-cni,kind=host,source=/var/lib/cni \
+  --mount volume=var-lib-cni,target=/var/lib/cni \
+  --volume var-log,kind=host,source=/var/log \
+  --mount volume=var-log,target=/var/log"
+Environment="KUBELET_IMAGE_URL=quay.io/coreos/hyperkube" "KUBELET_IMAGE_TAG=${var.tectonic_version}"
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+ExecStartPre=/bin/mkdir -p /var/lib/cni
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
+ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
+  --cni-conf-dir=/etc/kubernetes/cni/net.d \
+  --network-plugin=cni \
+  --lock-file=/var/run/lock/kubelet.lock \
+  --exit-on-lock-contention \
+  --pod-manifest-path=/etc/kubernetes/manifests \
+  --allow-privileged=true \
+  --minimum-container-ttl-duration=6m0s \
+  --cluster_dns=10.3.0.10 \
+  --cluster_domain=cluster.local
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+EOF
+}
+
+resource "ignition_config" "worker" {
+  count    = "${var.worker_count}"
+
+  users = [
+    "${ignition_user.core.id}",
+  ]
+
+  files = [
+    "${ignition_file.worker_hostname.*.id[count.index]}",
+    "${ignition_file.worker_kubeconfig.id}",
+    "${ignition_file.worker_resolv_conf.id}",
+    "${ignition_file.worker_ca_pem.id}",
+    "${ignition_file.worker_client_pem.id}",
+    "${ignition_file.worker_client_key.id}",
+  ]
+
+  systemd = [
+    "${ignition_systemd_unit.worker_locksmithd.id}",
+    "${ignition_systemd_unit.worker_etcd-member.id}",
+    "${ignition_systemd_unit.worker_kubelet.id}",
+  ]
+}

openstack-novanet/master.tf

@@ -1,22 +1,22 @@
-resource "openstack_compute_instance_v2" "control_node" {
-  count           = "${var.controller_count}"
-  name            = "control_node_${count.index}"
+resource "openstack_compute_instance_v2" "master_node" {
+  count           = "${var.master_count}"
+  name            = "master_node_${count.index}"
   image_id        = "${var.image_id}"
   flavor_id       = "${var.flavor_id}"
   key_pair        = "${openstack_compute_keypair_v2.k8s_keypair.name}"
-  security_groups = ["${openstack_compute_secgroup_v2.k8s_control_group.name}"]
+  security_groups = ["${openstack_compute_secgroup_v2.k8s_master_group.name}"]
 
   metadata {
-    role = "controller"
+    role = "master"
   }
 
-  user_data    = "${data.template_file.userdata-master.*.rendered[count.index]}"
+  user_data    = "${ignition_config.master.*.rendered[count.index]}"
   config_drive = false
 }
 
-resource "openstack_compute_secgroup_v2" "k8s_control_group" {
-  name        = "k8s_control_group"
-  description = "security group for k8s controllers: SSH and https"
+resource "openstack_compute_secgroup_v2" "k8s_master_group" {
+  name        = "k8s_master_group"
+  description = "security group for k8s masters: SSH and https"
 
   rule {
     from_port   = 22
@@ -43,7 +43,7 @@ resource "openstack_compute_secgroup_v2" "k8s_control_group" {
 resource "null_resource" "copy_assets" {
   # Changes to any instance of the cluster requires re-provisioning
   triggers {
-    cluster_instance_ids = "${join(" ", openstack_compute_instance_v2.control_node.*.id)}"
+    cluster_instance_ids = "${join(" ", openstack_compute_instance_v2.master_node.*.id)}"
   }
 
   # Bootstrap script can run on any instance of the cluster
@@ -51,7 +51,7 @@ resource "null_resource" "copy_assets" {
   connection {
     user        = "core"
     private_key = "${tls_private_key.core.private_key_pem}"
-    host        = "${element(openstack_compute_instance_v2.control_node.*.access_ip_v4, 0)}"
+    host        = "${element(openstack_compute_instance_v2.master_node.*.access_ip_v4, 0)}"
   }
 
   provisioner "file" {

openstack-novanet/secrets.tf

@@ -3,7 +3,7 @@ resource "tls_private_key" "core" {
 }
 
 resource "openstack_compute_keypair_v2" "k8s_keypair" {
-  name       = "k8s_keypair"
+  name       = "${var.cluster_name}_keypair"
   public_key = "${tls_private_key.core.public_key_openssh}"
 }
 
@@ -16,3 +16,10 @@ resource "null_resource" "export" {
     command = "echo '${tls_private_key.core.public_key_openssh}' >id_rsa_core.pub"
   }
 }
+
+resource "ignition_user" "core" {
+  name                = "core"
+  ssh_authorized_keys = [
+    "${tls_private_key.core.public_key_openssh}",
+  ]
+}

openstack-novanet/userdata-etcd.yml

@@ -1,24 +0,0 @@
-#cloud-config
-
-coreos:
-  units:
-  - name: "etcd2.service"
-    enable: false
-  - name: "etcd.service"
-    enable: false
-  - name: "etcd-member.service"
-    enable: true
-    command: "start"
-    drop-ins: 
-    - name: "40-etcd-cluster.conf"
-      content: |
-        [Service]
-        Environment="ETCD_IMAGE_TAG=v3.1.0"
-        ExecStart=
-        ExecStart=/usr/lib/coreos/etcd-wrapper \
-        --name=etcd \
-        --advertise-client-urls=http://${COREOS_PRIVATE_IPV4}:2379 \
-        --initial-advertise-peer-urls=http://${COREOS_PRIVATE_IPV4}:2380 \
-        --listen-client-urls=http://0.0.0.0:2379 \
-        --listen-peer-urls=http://0.0.0.0:2380 \
-        --initial-cluster=etcd=http://${COREOS_PRIVATE_IPV4}:2380

openstack-novanet/userdata-master.tf

@@ -1,15 +0,0 @@
-data "template_file" "userdata-master" {
-  count    = "${var.worker_count}"
-  template = "${file("${path.module}/userdata-master.yml")}"
-
-  vars {
-    kube_config      = "${base64encode(file("${path.root}/../assets/auth/kubeconfig"))}"
-    tectonic_version = "${var.tectonic_version}"
-    etcd_fqdn        = "${aws_route53_record.etcd.fqdn}"
-    ca               = "${base64encode(file("${path.root}/../assets/tls/ca.crt"))}"
-    client_crt       = "${base64encode(file("${path.root}/../assets/tls/kubelet.crt"))}"
-    client_crt_key   = "${base64encode(file("${path.root}/../assets/tls/kubelet.key"))}"
-    node_hostname    = "${var.cluster_name}-controller-${count.index}"
-    base_domain      = "${var.base_domain}"
-  }
-}

openstack-novanet/userdata-master.yml

@@ -1,122 +0,0 @@
-#cloud-config
-
-hostname: "${node_hostname}"
-
-write_files:
-  - path: "/opt/bootkube/.empty"
-    permissions: "0420"
-    owner: "root"
-    content: ""
-  - path: "/etc/kubernetes/kubelet.env"
-    permissions: "0644"
-    owner: "root"
-    content: "KUBELET_IMAGE_URL=quay.io/coreos/hyperkube KUBELET_IMAGE_TAG=v1.5.2_coreos.1"
-  - path: "/etc/kubernetes/kubeconfig"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${kube_config}
-  - path: "/etc/sysctl.d/max-user-watches.conf"
-    permissions: "0644"
-    owner: "root"
-    content: "fs.inotify.max_user_watches=16184"
-  - path: "/etc/kubernetes/ssl/ca.pem"
-    permissions: "0644"
-    owner: "root"
-    content: ${ca}
-    encoding: "base64"
-  - path: "/etc/kubernetes/ssl/client.pem"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${client_crt}
-  - path: "/etc/kubernetes/ssl/client-key.pem"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${client_crt_key}
-  - path: "/etc/resolv.conf"
-    permissions: "0644"
-    owner: "root"
-    content: |
-      search ${base_domain}
-      nameserver 8.8.8.8
-      nameserver 8.8.4.4
-
-coreos:
-  update:
-    reboot-strategy: "off"
-  locksmith:
-    endpoint: "http://localhost:2379"
-  units:
-  - name: etcd-member.service
-    command: "start"
-    enable: true
-    drop-ins:
-      - name: 40-etcd-gateway.conf
-        content: |
-          [Service]
-          Type=simple
-          Environment="ETCD_IMAGE_TAG=v3.1.0"
-          ExecStart=
-          ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \
-                --listen-addr=127.0.0.1:2379 \
-                --endpoints=${etcd_fqdn}:2379
-
-  - name: locksmithd.service
-    enable: false
-
-
-  - name: "bootkube.service"
-    enable: false
-    content: |
-      [Unit]
-      Description=Bootstrap a Kubernetes control plane with a temp api-server
-
-      [Service]
-      Type=oneshot
-      WorkingDirectory=/opt/bootkube
-      ExecStartPre=-chmod a+x /opt/bootkube/assets/bootkube-start
-      ExecStart=/opt/bootkube/assets/bootkube-start
-
-  - name: "kubelet.service"
-    command: "start"
-    enable: true
-    content: |
-      [Unit]
-      Description=Kubelet via Hyperkube ACI
-
-      [Service]
-      Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
-        --volume=resolv,kind=host,source=/etc/resolv.conf \
-        --mount volume=resolv,target=/etc/resolv.conf \
-        --volume var-lib-cni,kind=host,source=/var/lib/cni \
-        --mount volume=var-lib-cni,target=/var/lib/cni \
-        --volume var-log,kind=host,source=/var/log \
-        --mount volume=var-log,target=/var/log"
-      Environment="KUBELET_IMAGE_URL=quay.io/coreos/hyperkube" "KUBELET_IMAGE_TAG=${tectonic_version}"
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
-      ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
-      ExecStartPre=/bin/mkdir -p /var/lib/cni
-      ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
-      ExecStart=/usr/lib/coreos/kubelet-wrapper \
-        --kubeconfig=/etc/kubernetes/kubeconfig \
-        --require-kubeconfig \
-        --cni-conf-dir=/etc/kubernetes/cni/net.d \
-        --network-plugin=cni \
-        --lock-file=/var/run/lock/kubelet.lock \
-        --exit-on-lock-contention \
-        --pod-manifest-path=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --node-labels=master=true \
-        --minimum-container-ttl-duration=6m0s \
-        --cluster_dns=10.3.0.10 \
-        --cluster_domain=cluster.local
-      ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
-      Restart=always
-      RestartSec=10
-
-      [Install]
-      WantedBy=multi-user.target

openstack-novanet/userdata-worker.tf

@@ -1,15 +0,0 @@
-data "template_file" "userdata-worker" {
-  count    = "${var.worker_count}"
-  template = "${file("${path.module}/userdata-worker.yml")}"
-
-  vars {
-    kube_config      = "${base64encode(file("${path.root}/../assets/auth/kubeconfig"))}"
-    tectonic_version = "${var.tectonic_version}"
-    etcd_fqdn        = "${aws_route53_record.etcd.fqdn}"
-    ca               = "${base64encode(file("${path.root}/../assets/tls/ca.crt"))}"
-    client_crt       = "${base64encode(file("${path.root}/../assets/tls/kubelet.crt"))}"
-    client_crt_key   = "${base64encode(file("${path.root}/../assets/tls/kubelet.key"))}"
-    node_hostname    = "${var.cluster_name}-worker-${count.index}"
-    base_domain      = "${var.base_domain}"
-  }
-}

openstack-novanet/userdata-worker.yml

@@ -1,90 +0,0 @@
-#cloud-config
-
-hostname: "${node_hostname}"
-
-write_files:
-  - path: "/etc/kubernetes/kubeconfig"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${kube_config}
-  - path: "/etc/kubernetes/ssl/ca.pem"
-    permissions: "0644"
-    owner: "root"
-    content: ${ca}
-    encoding: "base64"
-  - path: "/etc/kubernetes/ssl/client.pem"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${client_crt}
-  - path: "/etc/kubernetes/ssl/client-key.pem"
-    permissions: "0644"
-    owner: "root"
-    encoding: "base64"
-    content: ${client_crt_key}
-  - path: /etc/resolv.conf
-    permissions: "0644"
-    owner: "root"
-    content: |
-      search ${base_domain}
-      nameserver 8.8.8.8
-      nameserver 8.8.4.4
-
-coreos:
-  update:
-    reboot-strategy: "off"
-  locksmith:
-    endpoint: "http://${etcd_fqdn}:2379"
-  units:
-  - name: etcd-member.service
-    enable: true
-    drop-ins:
-      - name: 40-etcd-gateway.conf
-        content: |
-          [Service]
-          Environment="ETCD_IMAGE_TAG=v3.1.0"
-          ExecStart=
-          ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \
-              --listen-addr=127.0.0.1:2379 \
-              --endpoints=${etcd_fqdn}:2379
-  - name: "kubelet.service"
-    command: "start"
-    enable: true
-    content: |
-      [Unit]
-      Description=Kubelet via Hyperkube ACI
-
-      [Service]
-      Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \
-        --volume=resolv,kind=host,source=/etc/resolv.conf \
-        --mount volume=resolv,target=/etc/resolv.conf \
-        --volume var-lib-cni,kind=host,source=/var/lib/cni \
-        --mount volume=var-lib-cni,target=/var/lib/cni \
-        --volume var-log,kind=host,source=/var/log \
-        --mount volume=var-log,target=/var/log"
-      Environment="KUBELET_IMAGE_URL=quay.io/coreos/hyperkube" "KUBELET_IMAGE_TAG=${tectonic_version}"
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
-      ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
-      ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
-      ExecStartPre=/bin/mkdir -p /var/lib/cni
-      ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
-      ExecStart=/usr/lib/coreos/kubelet-wrapper \
-        --kubeconfig=/etc/kubernetes/kubeconfig \
-        --require-kubeconfig \
-        --cni-conf-dir=/etc/kubernetes/cni/net.d \
-        --network-plugin=cni \
-        --lock-file=/var/run/lock/kubelet.lock \
-        --exit-on-lock-contention \
-        --pod-manifest-path=/etc/kubernetes/manifests \
-        --allow-privileged=true \
-        --minimum-container-ttl-duration=6m0s \
-        --cluster_dns=10.3.0.10 \
-        --cluster_domain=cluster.local
-      ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid
-      Restart=always
-      RestartSec=10
-
-      [Install]
-      WantedBy=multi-user.target

openstack-novanet/workers.tf

@@ -9,6 +9,6 @@ resource "openstack_compute_instance_v2" "worker_node" {
     role = "worker"
   }
 
-  user_data    = "${data.template_file.userdata-worker.*.rendered[count.index]}"
+  user_data    = "${ignition_config.worker.*.rendered[count.index]}"
   config_drive = false
 }