From 27c03c8ead0ebd13efdde0d0c2498194be651b8e Mon Sep 17 00:00:00 2001
From: Joseph Callen <jcallen@redhat.com>
Date: Wed, 8 Apr 2020 12:47:02 -0400
Subject: [PATCH] vsphere upi: terraform update, general updates and reorg

- Update to terraform 0.12.x
- ipam moved to a module and updated for 0.12
- creation of vsphere objects moved to module
- removal of etcd dns records
- domain dns records separated from creating rhcos node a records
- create load balancer using openshift haproxy image and an additional
rhcos virtual machine
- create dns A record for lb ip and api, api-int, *.apps
- change virtual machine ignition extra config to file path string vs
ignition string.  Simplifies bootstrap and other instance configuration
- Updated Dockerfile CI UPI image for 0.12.24
---
 images/installer/Dockerfile.upi.ci          |   2 +-
 upi/vsphere/cluster_domain/main.tf          |  22 ++
 upi/vsphere/cluster_domain/outputs.tf       |   3 +
 upi/vsphere/cluster_domain/variables.tf     |   9 +
 upi/vsphere/folder/main.tf                  |   5 -
 upi/vsphere/folder/output.tf                |   3 -
 upi/vsphere/folder/variables.tf             |   7 -
 upi/vsphere/host_a_record/main.tf           |   9 +
 upi/vsphere/host_a_record/outputs.tf        |   3 +
 upi/vsphere/host_a_record/variables.tf      |   9 +
 upi/vsphere/{machine => ipam}/cidr_to_ip.sh |   6 +-
 upi/vsphere/ipam/main.tf                    |  41 +++
 upi/vsphere/ipam/outputs.tf                 |   3 +
 upi/vsphere/ipam/variables.tf               |  20 ++
 upi/vsphere/ipam/versions.tf                |   3 +
 upi/vsphere/lb/haproxy.service              |  20 ++
 upi/vsphere/lb/haproxy.tmpl                 |  55 ++++
 upi/vsphere/lb/main.tf                      |  29 ++
 upi/vsphere/lb/outputs.tf                   |   4 +
 upi/vsphere/lb/variables.tf                 |  15 +
 upi/vsphere/machine/ignition.tf             |  79 -----
 upi/vsphere/machine/ip.tf                   |  42 ---
 upi/vsphere/machine/main.tf                 |  51 ----
 upi/vsphere/machine/output.tf               |   3 -
 upi/vsphere/machine/variables.tf            |  68 -----
 upi/vsphere/main.tf                         | 306 +++++++++++++++-----
 upi/vsphere/resource_pool/main.tf           |   9 -
 upi/vsphere/resource_pool/output.tf         |   3 -
 upi/vsphere/resource_pool/variables.tf      |  11 -
 upi/vsphere/route53/main.tf                 |  92 ------
 upi/vsphere/route53/variables.tf            |  33 ---
 upi/vsphere/variables.tf                    | 175 ++++++-----
 upi/vsphere/vm/ifcfg.tmpl                   |  12 +
 upi/vsphere/vm/ignition.tf                  |  47 +++
 upi/vsphere/vm/main.tf                      |  36 +++
 upi/vsphere/vm/variables.tf                 |  61 ++++
 upi/vsphere/vm/versions.tf                  |   3 +
 37 files changed, 731 insertions(+), 568 deletions(-)
 create mode 100644 upi/vsphere/cluster_domain/main.tf
 create mode 100644 upi/vsphere/cluster_domain/outputs.tf
 create mode 100644 upi/vsphere/cluster_domain/variables.tf
 delete mode 100644 upi/vsphere/folder/main.tf
 delete mode 100644 upi/vsphere/folder/output.tf
 delete mode 100644 upi/vsphere/folder/variables.tf
 create mode 100644 upi/vsphere/host_a_record/main.tf
 create mode 100644 upi/vsphere/host_a_record/outputs.tf
 create mode 100644 upi/vsphere/host_a_record/variables.tf
 rename upi/vsphere/{machine => ipam}/cidr_to_ip.sh (99%)
 create mode 100644 upi/vsphere/ipam/main.tf
 create mode 100644 upi/vsphere/ipam/outputs.tf
 create mode 100644 upi/vsphere/ipam/variables.tf
 create mode 100644 upi/vsphere/ipam/versions.tf
 create mode 100644 upi/vsphere/lb/haproxy.service
 create mode 100644 upi/vsphere/lb/haproxy.tmpl
 create mode 100644 upi/vsphere/lb/main.tf
 create mode 100644 upi/vsphere/lb/outputs.tf
 create mode 100644 upi/vsphere/lb/variables.tf
 delete mode 100644 upi/vsphere/machine/ignition.tf
 delete mode 100644 upi/vsphere/machine/ip.tf
 delete mode 100644 upi/vsphere/machine/main.tf
 delete mode 100644 upi/vsphere/machine/output.tf
 delete mode 100644 upi/vsphere/machine/variables.tf
 delete mode 100644 upi/vsphere/resource_pool/main.tf
 delete mode 100644 upi/vsphere/resource_pool/output.tf
 delete mode 100644 upi/vsphere/resource_pool/variables.tf
 delete mode 100644 upi/vsphere/route53/main.tf
 delete mode 100644 upi/vsphere/route53/variables.tf
 create mode 100644 upi/vsphere/vm/ifcfg.tmpl
 create mode 100644 upi/vsphere/vm/ignition.tf
 create mode 100644 upi/vsphere/vm/main.tf
 create mode 100644 upi/vsphere/vm/variables.tf
 create mode 100644 upi/vsphere/vm/versions.tf

diff --git a/images/installer/Dockerfile.upi.ci b/images/installer/Dockerfile.upi.ci
index bfad7d212f..4c9067996b 100644
--- a/images/installer/Dockerfile.upi.ci
+++ b/images/installer/Dockerfile.upi.ci
@@ -30,7 +30,7 @@ RUN yum install --setopt=tsflags=nodocs -y \
     yum clean all && rm -rf /var/cache/yum/* && \
     chmod g+w /etc/passwd
 
-ENV TERRAFORM_VERSION=0.11.11
+ENV TERRAFORM_VERSION=0.12.24
 RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
     unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /bin/
 ENV MATCHBOX_VERSION=v0.2.3
diff --git a/upi/vsphere/cluster_domain/main.tf b/upi/vsphere/cluster_domain/main.tf
new file mode 100644
index 0000000000..26814e8150
--- /dev/null
+++ b/upi/vsphere/cluster_domain/main.tf
@@ -0,0 +1,22 @@
+data "aws_route53_zone" "base" {
+  name = var.base_domain
+}
+
+resource "aws_route53_zone" "cluster" {
+  name          = var.cluster_domain
+  force_destroy = true
+
+  tags = {
+    "Name"     = var.cluster_domain
+    "Platform" = "vSphere"
+  }
+}
+
+resource "aws_route53_record" "name_server" {
+  name    = var.cluster_domain
+  type    = "NS"
+  ttl     = "300"
+  zone_id = data.aws_route53_zone.base.zone_id
+  records = aws_route53_zone.cluster.name_servers
+}
+
diff --git a/upi/vsphere/cluster_domain/outputs.tf b/upi/vsphere/cluster_domain/outputs.tf
new file mode 100644
index 0000000000..ef8db09d4c
--- /dev/null
+++ b/upi/vsphere/cluster_domain/outputs.tf
@@ -0,0 +1,3 @@
+output "zone_id" {
+  value = aws_route53_zone.cluster.zone_id
+}
diff --git a/upi/vsphere/cluster_domain/variables.tf b/upi/vsphere/cluster_domain/variables.tf
new file mode 100644
index 0000000000..83699e5c09
--- /dev/null
+++ b/upi/vsphere/cluster_domain/variables.tf
@@ -0,0 +1,9 @@
+variable "cluster_domain" {
+  description = "The domain for the cluster that all DNS records must belong"
+  type        = string
+}
+
+variable "base_domain" {
+  description = "The base domain used for public records."
+  type        = string
+}
diff --git a/upi/vsphere/folder/main.tf b/upi/vsphere/folder/main.tf
deleted file mode 100644
index 6f5605846f..0000000000
--- a/upi/vsphere/folder/main.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-resource "vsphere_folder" "folder" {
-  path          = "${var.path}"
-  type          = "vm"
-  datacenter_id = "${var.datacenter_id}"
-}
diff --git a/upi/vsphere/folder/output.tf b/upi/vsphere/folder/output.tf
deleted file mode 100644
index d20b194905..0000000000
--- a/upi/vsphere/folder/output.tf
+++ /dev/null
@@ -1,3 +0,0 @@
-output "path" {
-  value = "${vsphere_folder.folder.path}"
-}
diff --git a/upi/vsphere/folder/variables.tf b/upi/vsphere/folder/variables.tf
deleted file mode 100644
index a02bf0cfbc..0000000000
--- a/upi/vsphere/folder/variables.tf
+++ /dev/null
@@ -1,7 +0,0 @@
-variable "path" {
-  type = "string"
-}
-
-variable "datacenter_id" {
-  type = "string"
-}
diff --git a/upi/vsphere/host_a_record/main.tf b/upi/vsphere/host_a_record/main.tf
new file mode 100644
index 0000000000..8ffa864c06
--- /dev/null
+++ b/upi/vsphere/host_a_record/main.tf
@@ -0,0 +1,9 @@
+resource "aws_route53_record" "a_record" {
+  for_each = var.records
+
+  type    = "A"
+  ttl     = "60"
+  zone_id = var.zone_id
+  name    = each.key
+  records = [each.value]
+}
diff --git a/upi/vsphere/host_a_record/outputs.tf b/upi/vsphere/host_a_record/outputs.tf
new file mode 100644
index 0000000000..091a402885
--- /dev/null
+++ b/upi/vsphere/host_a_record/outputs.tf
@@ -0,0 +1,3 @@
+output "fqdns" {
+  value = values(aws_route53_record.a_record)[*].name
+}
diff --git a/upi/vsphere/host_a_record/variables.tf b/upi/vsphere/host_a_record/variables.tf
new file mode 100644
index 0000000000..710ed1e1fd
--- /dev/null
+++ b/upi/vsphere/host_a_record/variables.tf
@@ -0,0 +1,9 @@
+variable "zone_id" {
+  type        = string
+  description = "The ID of the hosted zone to contain this record."
+}
+
+variable "records" {
+  type        = map(string)
+  description = "A records to be added to the zone_id"
+}
diff --git a/upi/vsphere/machine/cidr_to_ip.sh b/upi/vsphere/ipam/cidr_to_ip.sh
similarity index 99%
rename from upi/vsphere/machine/cidr_to_ip.sh
rename to upi/vsphere/ipam/cidr_to_ip.sh
index 9bfb742d7b..11cd84a3d6 100755
--- a/upi/vsphere/machine/cidr_to_ip.sh
+++ b/upi/vsphere/ipam/cidr_to_ip.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# cidr_to_ip - 
+# cidr_to_ip -
 #  https://www.terraform.io/docs/providers/external/data_source.html
 #  Based on info from here: https://gist.github.com/irvingpop/968464132ded25a206ced835d50afa6b
 #  This script takes requests an IP address from an IPAM server
@@ -58,11 +58,11 @@ function produce_output() {
   # The verification and looping is a crude way of overcoming the lack of
   # currency safety in the IPAM server.
   while [[ $SECONDS -lt $timeout ]]
-  do 
+  do
     ip_address=$(curl -s "http://$ipam/api/getFreeIP.php?apiapp=address&apitoken=$ipam_token&subnet=${network}&host=${hostname}")
 
     if [[ "$(is_ip_address "${ip_address}")" != "true" ]]; then error_exit "could not reserve an IP address: ${ip_address}"; fi
-	
+
     if [[ "$ip_address" == "$(get_reservation)" ]]
 	then
       jq -n \
diff --git a/upi/vsphere/ipam/main.tf b/upi/vsphere/ipam/main.tf
new file mode 100644
index 0000000000..13501ea961
--- /dev/null
+++ b/upi/vsphere/ipam/main.tf
@@ -0,0 +1,41 @@
+locals {
+  network      = cidrhost(var.machine_cidr, 0)
+  hostnames    = length(var.static_ip_addresses) == 0 ? var.hostnames : []
+  ip_addresses = length(var.static_ip_addresses) == 0 ? [for result in null_resource.ip_address : jsondecode(data.http.getip[result.triggers.hostname].body)[result.triggers.hostname]] : var.static_ip_addresses
+}
+
+data "http" "getip" {
+  for_each = null_resource.ip_address
+
+  url = "http://${var.ipam}/api/getIPs.php?apiapp=address&apitoken=${var.ipam_token}&domain=${null_resource.ip_address[each.key].triggers.hostname}"
+
+  request_headers = {
+    Accept = "application/json"
+  }
+}
+
+resource "null_resource" "ip_address" {
+  for_each = local.hostnames
+
+  triggers = {
+    ipam       = var.ipam
+    ipam_token = var.ipam_token
+    network    = local.network
+    hostname   = each.key
+  }
+
+  provisioner "local-exec" {
+    command = <<EOF
+echo '{"network":"${self.triggers.network}","hostname":"${self.triggers.hostname}","ipam":"${self.triggers.ipam}","ipam_token":"${self.triggers.ipam_token}"}' | ${path.module}/cidr_to_ip.sh
+EOF
+
+  }
+  provisioner "local-exec" {
+    when = destroy
+
+    command = <<EOF
+curl -s "http://${self.triggers.ipam}/api/removeHost.php?apiapp=address&apitoken=${self.triggers.ipam_token}&host=${self.triggers.hostname}"
+EOF
+
+  }
+}
diff --git a/upi/vsphere/ipam/outputs.tf b/upi/vsphere/ipam/outputs.tf
new file mode 100644
index 0000000000..3cc5fa5bb4
--- /dev/null
+++ b/upi/vsphere/ipam/outputs.tf
@@ -0,0 +1,3 @@
+output "ip_addresses" {
+  value = local.ip_addresses
+}
diff --git a/upi/vsphere/ipam/variables.tf b/upi/vsphere/ipam/variables.tf
new file mode 100644
index 0000000000..37cb664b84
--- /dev/null
+++ b/upi/vsphere/ipam/variables.tf
@@ -0,0 +1,20 @@
+variable "hostnames" {
+  type = set(string)
+}
+
+variable "machine_cidr" {
+  type = string
+}
+
+variable "ipam" {
+  type = string
+}
+
+variable "ipam_token" {
+  type = string
+}
+
+variable "static_ip_addresses" {
+  type    = list(string)
+  default = []
+}
diff --git a/upi/vsphere/ipam/versions.tf b/upi/vsphere/ipam/versions.tf
new file mode 100644
index 0000000000..d9b6f790b9
--- /dev/null
+++ b/upi/vsphere/ipam/versions.tf
@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}
diff --git a/upi/vsphere/lb/haproxy.service b/upi/vsphere/lb/haproxy.service
new file mode 100644
index 0000000000..648598fe1c
--- /dev/null
+++ b/upi/vsphere/lb/haproxy.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=haproxy
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+TimeoutStartSec=0
+ExecStartPre=-/bin/podman kill haproxy
+ExecStartPre=-/bin/podman rm haproxy
+ExecStartPre=/bin/podman pull quay.io/openshift/origin-haproxy-router
+ExecStart=/bin/podman run --name haproxy \
+  --net=host \
+  --privileged \
+  --entrypoint=/usr/sbin/haproxy \
+  -v /etc/haproxy/haproxy.conf:/var/lib/haproxy/conf/haproxy.conf:Z \
+  quay.io/openshift/origin-haproxy-router -f /var/lib/haproxy/conf/haproxy.conf
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/upi/vsphere/lb/haproxy.tmpl b/upi/vsphere/lb/haproxy.tmpl
new file mode 100644
index 0000000000..06cfb21036
--- /dev/null
+++ b/upi/vsphere/lb/haproxy.tmpl
@@ -0,0 +1,55 @@
+defaults
+  maxconn 20000
+  mode    tcp
+  log     /var/run/haproxy/haproxy-log.sock local0
+  option  dontlognull
+  retries 3
+  timeout http-request 10s
+  timeout queue        1m
+  timeout connect      10s
+  timeout client       86400s
+  timeout server       86400s
+  timeout tunnel       86400s
+
+frontend api-server
+    bind ${lb_ip_address}:6443
+    default_backend api-server
+
+frontend machine-config-server
+    bind ${lb_ip_address}:22623
+    default_backend machine-config-server
+
+frontend router-http
+    bind ${lb_ip_address}:80
+    default_backend router-http
+
+frontend router-https
+    bind ${lb_ip_address}:443
+    default_backend router-https
+
+backend api-server
+    balance roundrobin
+    %{ for addr in api ~}
+    server ${addr} ${addr}:6443 check
+    %{ endfor ~}
+
+backend machine-config-server
+    balance roundrobin
+    %{ for addr in api ~}
+    server ${addr} ${addr}:22623 check
+    %{ endfor ~}
+
+backend router-http
+    balance source
+    mode tcp
+    %{ for addr in ingress ~}
+    server ${addr} ${addr}:80 check
+    %{ endfor ~}
+
+backend router-https
+    balance source
+    mode tcp
+    %{ for addr in ingress ~}
+    server ${addr} ${addr}:443 check
+    %{ endfor ~}
+
diff --git a/upi/vsphere/lb/main.tf b/upi/vsphere/lb/main.tf
new file mode 100644
index 0000000000..7784a0e78b
--- /dev/null
+++ b/upi/vsphere/lb/main.tf
@@ -0,0 +1,29 @@
+data "ignition_systemd_unit" "haproxy" {
+  name    = "haproxy.service"
+  content = file("${path.module}/haproxy.service")
+}
+
+data "ignition_file" "haproxy" {
+  filesystem = "root"
+  path       = "/etc/haproxy/haproxy.conf"
+  mode       = 0755
+  content {
+    content = templatefile("${path.module}/haproxy.tmpl", {
+      lb_ip_address = var.lb_ip_address,
+      api           = var.api_backend_addresses,
+      ingress       = var.ingress_backend_addresses
+    })
+  }
+}
+
+data "ignition_user" "core" {
+  name                = "core"
+  ssh_authorized_keys = [file("${var.ssh_public_key_path}")]
+}
+
+data "ignition_config" "lb" {
+  users   = [data.ignition_user.core.rendered]
+  files   = [data.ignition_file.haproxy.rendered]
+  systemd = [data.ignition_systemd_unit.haproxy.rendered]
+}
+
diff --git a/upi/vsphere/lb/outputs.tf b/upi/vsphere/lb/outputs.tf
new file mode 100644
index 0000000000..e2d88b266f
--- /dev/null
+++ b/upi/vsphere/lb/outputs.tf
@@ -0,0 +1,4 @@
+output "ignition" {
+  value = data.ignition_config.lb.rendered
+}
+
diff --git a/upi/vsphere/lb/variables.tf b/upi/vsphere/lb/variables.tf
new file mode 100644
index 0000000000..358ae56671
--- /dev/null
+++ b/upi/vsphere/lb/variables.tf
@@ -0,0 +1,15 @@
+variable "lb_ip_address" {
+  type = string
+}
+
+variable "api_backend_addresses" {
+  type = list(string)
+}
+
+variable "ingress_backend_addresses" {
+  type = list(string)
+}
+
+variable "ssh_public_key_path" {
+  type = string
+}
diff --git a/upi/vsphere/machine/ignition.tf b/upi/vsphere/machine/ignition.tf
deleted file mode 100644
index 128bff7dd3..0000000000
--- a/upi/vsphere/machine/ignition.tf
+++ /dev/null
@@ -1,79 +0,0 @@
-provider "ignition" {
-  version = "1.1.0"
-}
-
-locals {
-  mask = "${element(split("/", var.machine_cidr), 1)}"
-  gw   = "${cidrhost(var.machine_cidr,1)}"
-
-  ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
-}
-
-data "ignition_file" "hostname" {
-  count = "${var.instance_count}"
-
-  filesystem = "root"
-  path       = "/etc/hostname"
-  mode       = "420"
-
-  content {
-    content = "${var.name}-${count.index}"
-  }
-}
-
-data "ignition_file" "static_ip" {
-  count = "${var.instance_count}"
-
-  filesystem = "root"
-  path       = "/etc/sysconfig/network-scripts/ifcfg-ens192"
-  mode       = "420"
-
-  content {
-    content = <<EOF
-TYPE=Ethernet
-BOOTPROTO=none
-NAME=ens192
-DEVICE=ens192
-ONBOOT=yes
-IPADDR=${local.ip_addresses[count.index]}
-PREFIX=${local.mask}
-GATEWAY=${local.gw}
-DOMAIN=${var.cluster_domain}
-DNS1=1.1.1.1
-DNS2=9.9.9.9
-EOF
-  }
-}
-
-data "ignition_systemd_unit" "restart" {
-  count = "${var.instance_count}"
-
-  name = "restart.service"
-
-  content = <<EOF
-[Unit]
-ConditionFirstBoot=yes
-[Service]
-Type=idle
-ExecStart=/sbin/reboot
-[Install]
-WantedBy=multi-user.target
-EOF
-}
-
-data "ignition_config" "ign" {
-  count = "${var.instance_count}"
-
-  append {
-    source = "${var.ignition_url != "" ? var.ignition_url : local.ignition_encoded}"
-  }
-
-  systemd = [
-    "${data.ignition_systemd_unit.restart.*.id[count.index]}",
-  ]
-
-  files = [
-    "${data.ignition_file.hostname.*.id[count.index]}",
-    "${data.ignition_file.static_ip.*.id[count.index]}",
-  ]
-}
diff --git a/upi/vsphere/machine/ip.tf b/upi/vsphere/machine/ip.tf
deleted file mode 100644
index 2ad92f8c4b..0000000000
--- a/upi/vsphere/machine/ip.tf
+++ /dev/null
@@ -1,42 +0,0 @@
-locals {
-  network      = "${cidrhost(var.machine_cidr,0)}"
-  ip_addresses = ["${coalescelist(var.ip_addresses, data.template_file.ip_address.*.rendered)}"]
-}
-
-data "external" "ip_address" {
-  count = "${length(var.ip_addresses) == 0 ? var.instance_count : 0}"
-
-  program = ["bash", "${path.module}/cidr_to_ip.sh"]
-
-  query = {
-    hostname   = "${var.name}-${count.index}.${var.cluster_domain}"
-    ipam       = "${var.ipam}"
-    ipam_token = "${var.ipam_token}"
-  }
-
-  depends_on = ["null_resource.ip_address"]
-}
-
-data "template_file" "ip_address" {
-  count = "${length(var.ip_addresses) == 0 ? var.instance_count : 0}"
-
-  template = "${lookup(data.external.ip_address.*.result[count.index], "ip_address")}"
-}
-
-resource "null_resource" "ip_address" {
-  count = "${length(var.ip_addresses) == 0 ? var.instance_count : 0}"
-
-  provisioner "local-exec" {
-    command = <<EOF
-echo '{"network":"${local.network}","hostname":"${var.name}-${count.index}.${var.cluster_domain}","ipam":"${var.ipam}","ipam_token":"${var.ipam_token}"}' | ${path.module}/cidr_to_ip.sh
-EOF
-  }
-
-  provisioner "local-exec" {
-    when = "destroy"
-
-    command = <<EOF
-curl -s "http://${var.ipam}/api/removeHost.php?apiapp=address&apitoken=${var.ipam_token}&host=${var.name}-${count.index}.${var.cluster_domain}"
-EOF
-  }
-}
diff --git a/upi/vsphere/machine/main.tf b/upi/vsphere/machine/main.tf
deleted file mode 100644
index ae49162a02..0000000000
--- a/upi/vsphere/machine/main.tf
+++ /dev/null
@@ -1,51 +0,0 @@
-data "vsphere_datastore" "datastore" {
-  name          = "${var.datastore}"
-  datacenter_id = "${var.datacenter_id}"
-}
-
-data "vsphere_network" "network" {
-  name          = "${var.network}"
-  datacenter_id = "${var.datacenter_id}"
-}
-
-data "vsphere_virtual_machine" "template" {
-  name          = "${var.template}"
-  datacenter_id = "${var.datacenter_id}"
-}
-
-resource "vsphere_virtual_machine" "vm" {
-  count = "${var.instance_count}"
-
-  name             = "${var.name}-${count.index}"
-  resource_pool_id = "${var.resource_pool_id}"
-  datastore_id     = "${data.vsphere_datastore.datastore.id}"
-  num_cpus         = "${var.num_cpu}"
-  memory           = "${var.memory}"
-  guest_id         = "${data.vsphere_virtual_machine.template.guest_id}"
-  folder           = "${var.folder}"
-  enable_disk_uuid = "true"
-
-  wait_for_guest_net_timeout  = "0"
-  wait_for_guest_net_routable = "false"
-
-  network_interface {
-    network_id = "${data.vsphere_network.network.id}"
-  }
-
-  disk {
-    label            = "disk0"
-    size             = 60
-    thin_provisioned = "${data.vsphere_virtual_machine.template.disks.0.thin_provisioned}"
-  }
-
-  clone {
-    template_uuid = "${data.vsphere_virtual_machine.template.id}"
-  }
-
-  vapp {
-    properties {
-      "guestinfo.ignition.config.data"          = "${base64encode(data.ignition_config.ign.*.rendered[count.index])}"
-      "guestinfo.ignition.config.data.encoding" = "base64"
-    }
-  }
-}
diff --git a/upi/vsphere/machine/output.tf b/upi/vsphere/machine/output.tf
deleted file mode 100644
index c62c1e2ac4..0000000000
--- a/upi/vsphere/machine/output.tf
+++ /dev/null
@@ -1,3 +0,0 @@
-output "ip_addresses" {
-  value = ["${local.ip_addresses}"]
-}
diff --git a/upi/vsphere/machine/variables.tf b/upi/vsphere/machine/variables.tf
deleted file mode 100644
index 4c8a13bee5..0000000000
--- a/upi/vsphere/machine/variables.tf
+++ /dev/null
@@ -1,68 +0,0 @@
-variable "name" {
-  type = "string"
-}
-
-variable "instance_count" {
-  type = "string"
-}
-
-variable "ignition" {
-  type    = "string"
-  default = ""
-}
-
-variable "ignition_url" {
-  type    = "string"
-  default = ""
-}
-
-variable "resource_pool_id" {
-  type = "string"
-}
-
-variable "folder" {
-  type = "string"
-}
-
-variable "datastore" {
-  type = "string"
-}
-
-variable "network" {
-  type = "string"
-}
-
-variable "cluster_domain" {
-  type = "string"
-}
-
-variable "datacenter_id" {
-  type = "string"
-}
-
-variable "template" {
-  type = "string"
-}
-
-variable "machine_cidr" {
-  type = "string"
-}
-
-variable "ipam" {
-  type = "string"
-}
-
-variable "ipam_token" {
-  type = "string"
-}
-
-variable "ip_addresses" {
-  type = "list"
-}
-variable "memory" {
-  type = "string"
-}
-
-variable "num_cpu" {
-  type = "string"
-}
diff --git a/upi/vsphere/main.tf b/upi/vsphere/main.tf
index b236a2226d..f1b9550b75 100644
--- a/upi/vsphere/main.tf
+++ b/upi/vsphere/main.tf
@@ -1,101 +1,247 @@
+locals {
+  bootstrap_fqdns     = ["bootstrap-0.${var.cluster_domain}"]
+  lb_fqdns            = ["lb-0.${var.cluster_domain}"]
+  api_lb_fqdns        = formatlist("%s.%s", ["api", "api-int", "*.apps"], var.cluster_domain)
+  control_plane_fqdns = [for idx in range(var.control_plane_count) : "control-plane-${idx}.${var.cluster_domain}"]
+  compute_fqdns       = [for idx in range(var.compute_count) : "compute-${idx}.${var.cluster_domain}"]
+}
+
 provider "vsphere" {
-  user                 = "${var.vsphere_user}"
-  password             = "${var.vsphere_password}"
-  vsphere_server       = "${var.vsphere_server}"
+  user                 = var.vsphere_user
+  password             = var.vsphere_password
+  vsphere_server       = var.vsphere_server
   allow_unverified_ssl = true
 }
 
 data "vsphere_datacenter" "dc" {
-  name = "${var.vsphere_datacenter}"
+  name = var.vsphere_datacenter
 }
 
-module "folder" {
-  source = "./folder"
-
-  path          = "${var.cluster_id}"
-  datacenter_id = "${data.vsphere_datacenter.dc.id}"
+data "vsphere_compute_cluster" "compute_cluster" {
+  name          = var.vsphere_cluster
+  datacenter_id = data.vsphere_datacenter.dc.id
 }
 
-module "resource_pool" {
-  source = "./resource_pool"
+data "vsphere_datastore" "datastore" {
+  name          = var.vsphere_datastore
+  datacenter_id = data.vsphere_datacenter.dc.id
+}
 
-  name            = "${var.cluster_id}"
-  datacenter_id   = "${data.vsphere_datacenter.dc.id}"
-  vsphere_cluster = "${var.vsphere_cluster}"
+data "vsphere_network" "network" {
+  name          = var.vm_network
+  datacenter_id = data.vsphere_datacenter.dc.id
+}
+
+data "vsphere_virtual_machine" "template" {
+  name          = var.vm_template
+  datacenter_id = data.vsphere_datacenter.dc.id
+}
+
+resource "vsphere_resource_pool" "resource_pool" {
+  name                    = var.cluster_id
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
+}
+
+resource "vsphere_folder" "folder" {
+  path          = var.cluster_id
+  type          = "vm"
+  datacenter_id = data.vsphere_datacenter.dc.id
+}
+
+// Request from phpIPAM a new IP address for the bootstrap node
+module "ipam_bootstrap" {
+  source = "./ipam"
+
+  // The hostname that will be added to phpIPAM when requesting an ip address
+  hostnames = local.bootstrap_fqdns
+
+  // Hostname or IP address of the phpIPAM server
+  ipam = var.ipam
+
+  // Access token for phpIPAM
+  ipam_token = var.ipam_token
+
+  // Subnet where we will request an ip address from phpIPAM
+  machine_cidr = var.machine_cidr
+
+  static_ip_addresses = var.bootstrap_ip_address == "" ? [] : [var.bootstrap_ip_address]
+
+}
+
+// Request from phpIPAM a new IP addresses for the control-plane nodes
+module "ipam_control_plane" {
+  source              = "./ipam"
+  hostnames           = local.control_plane_fqdns
+  ipam                = var.ipam
+  ipam_token          = var.ipam_token
+  machine_cidr        = var.machine_cidr
+  static_ip_addresses = var.control_plane_ip_addresses
+}
+
+// Request from phpIPAM a new IP addresses for the compute nodes
+module "ipam_compute" {
+  source              = "./ipam"
+  hostnames           = local.compute_fqdns
+  ipam                = var.ipam
+  ipam_token          = var.ipam_token
+  machine_cidr        = var.machine_cidr
+  static_ip_addresses = var.compute_ip_addresses
+}
+
+// Request from phpIPAM a new IP addresses for the load balancer nodes
+module "ipam_lb" {
+  source              = "./ipam"
+  hostnames           = local.lb_fqdns
+  ipam                = var.ipam
+  ipam_token          = var.ipam_token
+  machine_cidr        = var.machine_cidr
+  static_ip_addresses = var.lb_ip_address == "" ? [] : [var.lb_ip_address]
+}
+
+module "lb" {
+  source        = "./lb"
+  lb_ip_address = module.ipam_lb.ip_addresses[0]
+
+  api_backend_addresses = flatten([
+    module.ipam_bootstrap.ip_addresses[0],
+    module.ipam_control_plane.ip_addresses]
+  )
+
+  ingress_backend_addresses = module.ipam_compute.ip_addresses
+  ssh_public_key_path       = var.ssh_public_key_path
+}
+
+module "dns_cluster_domain" {
+  source         = "./cluster_domain"
+  cluster_domain = var.cluster_domain
+  base_domain    = var.base_domain
+}
+
+module "lb_a_records" {
+  source  = "./host_a_record"
+  zone_id = module.dns_cluster_domain.zone_id
+  records = zipmap(
+    local.api_lb_fqdns,
+    [for name in local.api_lb_fqdns : module.ipam_lb.ip_addresses[0]]
+  )
+}
+
+module "control_plane_a_records" {
+  source  = "./host_a_record"
+  zone_id = module.dns_cluster_domain.zone_id
+  records = zipmap(local.control_plane_fqdns, module.ipam_control_plane.ip_addresses)
+}
+
+module "compute_a_records" {
+  source  = "./host_a_record"
+  zone_id = module.dns_cluster_domain.zone_id
+  records = zipmap(local.compute_fqdns, module.ipam_compute.ip_addresses)
+}
+
+module "lb_vm" {
+  source = "./vm"
+
+  ignition               = module.lb.ignition
+  hostnames_ip_addresses = zipmap(local.lb_fqdns, module.ipam_lb.ip_addresses)
+
+  resource_pool_id      = vsphere_resource_pool.resource_pool.id
+  datastore_id          = data.vsphere_datastore.datastore.id
+  datacenter_id         = data.vsphere_datacenter.dc.id
+  network_id            = data.vsphere_network.network.id
+  folder_id             = vsphere_folder.folder.path
+  guest_id              = data.vsphere_virtual_machine.template.guest_id
+  template_uuid         = data.vsphere_virtual_machine.template.id
+  disk_thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
+
+  cluster_domain = var.cluster_domain
+  machine_cidr   = var.machine_cidr
+
+  num_cpus      = 2
+  memory        = 2096
+  dns_addresses = var.vm_dns_addresses
 }
 
 module "bootstrap" {
-  source = "./machine"
+  source = "./vm"
 
-  name             = "bootstrap"
-  instance_count   = "${var.bootstrap_complete ? 0 : 1}"
-  ignition_url     = "${var.bootstrap_ignition_url}"
-  resource_pool_id = "${module.resource_pool.pool_id}"
-  datastore        = "${var.vsphere_datastore}"
-  folder           = "${module.folder.path}"
-  network          = "${var.vm_network}"
-  datacenter_id    = "${data.vsphere_datacenter.dc.id}"
-  template         = "${var.vm_template}"
-  cluster_domain   = "${var.cluster_domain}"
-  ipam             = "${var.ipam}"
-  ipam_token       = "${var.ipam_token}"
-  ip_addresses     = ["${compact(list(var.bootstrap_ip))}"]
-  machine_cidr     = "${var.machine_cidr}"
-  memory           = "8192"
-  num_cpu          = "4"
+  ignition = file(var.bootstrap_ignition_path)
+
+  hostnames_ip_addresses = zipmap(
+    local.bootstrap_fqdns,
+    module.ipam_bootstrap.ip_addresses
+  )
+
+  resource_pool_id      = vsphere_resource_pool.resource_pool.id
+  datastore_id          = data.vsphere_datastore.datastore.id
+  datacenter_id         = data.vsphere_datacenter.dc.id
+  network_id            = data.vsphere_network.network.id
+  folder_id             = vsphere_folder.folder.path
+  guest_id              = data.vsphere_virtual_machine.template.guest_id
+  template_uuid         = data.vsphere_virtual_machine.template.id
+  disk_thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
+
+  cluster_domain = var.cluster_domain
+  machine_cidr   = var.machine_cidr
+
+  num_cpus      = 2
+  memory        = 8192
+  dns_addresses = var.vm_dns_addresses
 }
 
-module "control_plane" {
-  source = "./machine"
+module "control_plane_vm" {
+  source = "./vm"
 
-  name             = "control-plane"
-  instance_count   = "${var.control_plane_count}"
-  ignition         = "${var.control_plane_ignition}"
-  resource_pool_id = "${module.resource_pool.pool_id}"
-  folder           = "${module.folder.path}"
-  datastore        = "${var.vsphere_datastore}"
-  network          = "${var.vm_network}"
-  datacenter_id    = "${data.vsphere_datacenter.dc.id}"
-  template         = "${var.vm_template}"
-  cluster_domain   = "${var.cluster_domain}"
-  ipam             = "${var.ipam}"
-  ipam_token       = "${var.ipam_token}"
-  ip_addresses     = ["${var.control_plane_ips}"]
-  machine_cidr     = "${var.machine_cidr}"
-  memory           = "${var.master_memory}"
-  num_cpu          = "${var.master_num_cpus}"
+  // Using the output from control_plane_a_records
+  // is on purpose. I want the A records to be created before
+  // the virtual machines which gives additional time to
+  // replicate the records.
+  hostnames_ip_addresses = zipmap(
+    module.control_plane_a_records.fqdns,
+    module.ipam_control_plane.ip_addresses
+  )
+
+  ignition = file(var.control_plane_ignition_path)
+
+  resource_pool_id      = vsphere_resource_pool.resource_pool.id
+  datastore_id          = data.vsphere_datastore.datastore.id
+  datacenter_id         = data.vsphere_datacenter.dc.id
+  network_id            = data.vsphere_network.network.id
+  folder_id             = vsphere_folder.folder.path
+  guest_id              = data.vsphere_virtual_machine.template.guest_id
+  template_uuid         = data.vsphere_virtual_machine.template.id
+  disk_thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
+
+  cluster_domain = var.cluster_domain
+  machine_cidr   = var.machine_cidr
+
+  num_cpus      = var.control_plane_num_cpus
+  memory        = var.control_plane_memory
+  dns_addresses = var.vm_dns_addresses
 }
 
-module "compute" {
-  source = "./machine"
+module "compute_vm" {
+  source = "./vm"
 
-  name             = "compute"
-  instance_count   = "${var.compute_count}"
-  ignition         = "${var.compute_ignition}"
-  resource_pool_id = "${module.resource_pool.pool_id}"
-  folder           = "${module.folder.path}"
-  datastore        = "${var.vsphere_datastore}"
-  network          = "${var.vm_network}"
-  datacenter_id    = "${data.vsphere_datacenter.dc.id}"
-  template         = "${var.vm_template}"
-  cluster_domain   = "${var.cluster_domain}"
-  ipam             = "${var.ipam}"
-  ipam_token       = "${var.ipam_token}"
-  ip_addresses     = ["${var.compute_ips}"]
-  machine_cidr     = "${var.machine_cidr}"
-  memory           = "${var.compute_memory}"
-  num_cpu          = "${var.compute_num_cpus}"
-}
-
-module "dns" {
-  source = "./route53"
-
-  base_domain         = "${var.base_domain}"
-  cluster_domain      = "${var.cluster_domain}"
-  bootstrap_count     = "${var.bootstrap_complete ? 0 : 1}"
-  bootstrap_ips       = ["${module.bootstrap.ip_addresses}"]
-  control_plane_count = "${var.control_plane_count}"
-  control_plane_ips   = ["${module.control_plane.ip_addresses}"]
-  compute_count       = "${var.compute_count}"
-  compute_ips         = ["${module.compute.ip_addresses}"]
+  hostnames_ip_addresses = zipmap(
+    module.compute_a_records.fqdns,
+    module.ipam_compute.ip_addresses
+  )
+
+  ignition = file(var.compute_ignition_path)
+
+  resource_pool_id      = vsphere_resource_pool.resource_pool.id
+  datastore_id          = data.vsphere_datastore.datastore.id
+  datacenter_id         = data.vsphere_datacenter.dc.id
+  network_id            = data.vsphere_network.network.id
+  folder_id             = vsphere_folder.folder.path
+  guest_id              = data.vsphere_virtual_machine.template.guest_id
+  template_uuid         = data.vsphere_virtual_machine.template.id
+  disk_thin_provisioned = data.vsphere_virtual_machine.template.disks[0].thin_provisioned
+
+  cluster_domain = var.cluster_domain
+  machine_cidr   = var.machine_cidr
+
+  num_cpus      = var.compute_num_cpus
+  memory        = var.compute_memory
+  dns_addresses = var.vm_dns_addresses
 }
diff --git a/upi/vsphere/resource_pool/main.tf b/upi/vsphere/resource_pool/main.tf
deleted file mode 100644
index 3ed4ea77ab..0000000000
--- a/upi/vsphere/resource_pool/main.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-data "vsphere_compute_cluster" "compute_cluster" {
-  name          = "${var.vsphere_cluster}"
-  datacenter_id = "${var.datacenter_id}"
-}
-
-resource "vsphere_resource_pool" "resource_pool" {
-  name                    = "${var.name}"
-  parent_resource_pool_id = "${data.vsphere_compute_cluster.compute_cluster.resource_pool_id}"
-}
diff --git a/upi/vsphere/resource_pool/output.tf b/upi/vsphere/resource_pool/output.tf
deleted file mode 100644
index 68a5c26430..0000000000
--- a/upi/vsphere/resource_pool/output.tf
+++ /dev/null
@@ -1,3 +0,0 @@
-output "pool_id" {
-  value = "${vsphere_resource_pool.resource_pool.id}"
-}
diff --git a/upi/vsphere/resource_pool/variables.tf b/upi/vsphere/resource_pool/variables.tf
deleted file mode 100644
index a39933514a..0000000000
--- a/upi/vsphere/resource_pool/variables.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-variable "name" {
-  type = "string"
-}
-
-variable "datacenter_id" {
-  type = "string"
-}
-
-variable "vsphere_cluster" {
-  type = "string"
-}
diff --git a/upi/vsphere/route53/main.tf b/upi/vsphere/route53/main.tf
deleted file mode 100644
index 56b446d8b8..0000000000
--- a/upi/vsphere/route53/main.tf
+++ /dev/null
@@ -1,92 +0,0 @@
-data "aws_route53_zone" "base" {
-  name = "${var.base_domain}"
-}
-
-resource "aws_route53_zone" "cluster" {
-  name          = "${var.cluster_domain}"
-  force_destroy = true
-
-  tags = "${map(
-    "Name", "${var.cluster_domain}",
-	"Platform", "vSphere")}"
-}
-
-resource "aws_route53_record" "name_server" {
-  name    = "${var.cluster_domain}"
-  type    = "NS"
-  ttl     = "300"
-  zone_id = "${data.aws_route53_zone.base.zone_id}"
-  records = ["${aws_route53_zone.cluster.name_servers}"]
-}
-
-resource "aws_route53_record" "api-external" {
-  type           = "A"
-  ttl            = "60"
-  zone_id        = "${aws_route53_zone.cluster.zone_id}"
-  name           = "api.${var.cluster_domain}"
-  set_identifier = "api"
-  records        = ["${concat(var.bootstrap_ips, var.control_plane_ips)}"]
-
-  weighted_routing_policy {
-    weight = 90
-  }
-}
-
-resource "aws_route53_record" "api-internal" {
-  type           = "A"
-  ttl            = "60"
-  zone_id        = "${aws_route53_zone.cluster.zone_id}"
-  name           = "api-int.${var.cluster_domain}"
-  set_identifier = "api"
-  records        = ["${concat(var.bootstrap_ips, var.control_plane_ips)}"]
-
-  weighted_routing_policy {
-    weight = 90
-  }
-}
-
-resource "aws_route53_record" "etcd_a_nodes" {
-  count = "${var.control_plane_count}"
-
-  type    = "A"
-  ttl     = "60"
-  zone_id = "${aws_route53_zone.cluster.zone_id}"
-  name    = "etcd-${count.index}.${var.cluster_domain}"
-  records = ["${element(var.control_plane_ips, count.index)}"]
-}
-
-resource "aws_route53_record" "etcd_cluster" {
-  type    = "SRV"
-  ttl     = "60"
-  zone_id = "${aws_route53_zone.cluster.zone_id}"
-  name    = "_etcd-server-ssl._tcp"
-  records = ["${formatlist("0 10 2380 %s", aws_route53_record.etcd_a_nodes.*.fqdn)}"]
-}
-
-resource "aws_route53_record" "ingress" {
-  type    = "A"
-  ttl     = "60"
-  zone_id = "${aws_route53_zone.cluster.zone_id}"
-  name    = "*.apps.${var.cluster_domain}"
-  records = ["${var.compute_ips}"]
-}
-
-resource "aws_route53_record" "control_plane_nodes" {
-  count = "${var.control_plane_count}"
-
-  type    = "A"
-  ttl     = "60"
-  zone_id = "${aws_route53_zone.cluster.zone_id}"
-  name    = "control-plane-${count.index}.${var.cluster_domain}"
-  records = ["${element(var.control_plane_ips, count.index)}"]
-}
-
-resource "aws_route53_record" "compute_nodes" {
-  count = "${var.compute_count}"
-
-  type    = "A"
-  ttl     = "60"
-  zone_id = "${aws_route53_zone.cluster.zone_id}"
-  name    = "compute-${count.index}.${var.cluster_domain}"
-  records = ["${element(var.compute_ips, count.index)}"]
-}
diff --git a/upi/vsphere/route53/variables.tf b/upi/vsphere/route53/variables.tf
deleted file mode 100644
index 9c29ae3ad3..0000000000
--- a/upi/vsphere/route53/variables.tf
+++ /dev/null
@@ -1,33 +0,0 @@
-variable "cluster_domain" {
-  description = "The domain for the cluster that all DNS records must belong"
-  type        = "string"
-}
-
-variable "base_domain" {
-  description = "The base domain used for public records."
-  type        = "string"
-}
-
-variable "bootstrap_count" {
-  type = "string"
-}
-
-variable "bootstrap_ips" {
-  type = "list"
-}
-
-variable "control_plane_count" {
-  type = "string"
-}
-
-variable "control_plane_ips" {
-  type = "list"
-}
-
-variable "compute_count" {
-  type = "string"
-}
-
-variable "compute_ips" {
-  type = "list"
-}
diff --git a/upi/vsphere/variables.tf b/upi/vsphere/variables.tf
index 9a78fcca07..701f78de4c 100644
--- a/upi/vsphere/variables.tf
+++ b/upi/vsphere/variables.tf
@@ -1,153 +1,172 @@
-//////
-// vSphere variables
-//////
-
-variable "vsphere_server" {
-  type        = "string"
-  description = "This is the vSphere server for the environment."
-}
-
-variable "vsphere_user" {
-  type        = "string"
-  description = "vSphere server user for the environment."
-}
-
-variable "vsphere_password" {
-  type        = "string"
-  description = "vSphere server password"
-}
-
-variable "vsphere_cluster" {
-  type        = "string"
-  description = "This is the name of the vSphere cluster."
-}
-
-variable "vsphere_datacenter" {
-  type        = "string"
-  description = "This is the name of the vSphere data center."
-}
-
-variable "vsphere_datastore" {
-  type        = "string"
-  description = "This is the name of the vSphere data store."
-}
-
-variable "vm_template" {
-  type        = "string"
-  description = "This is the name of the VM template to clone."
-}
-
-variable "vm_network" {
-  type        = "string"
-  description = "This is the name of the publicly accessible network for cluster ingress and access."
-  default     = "VM Network"
-}
+// phpIPAM variables
 
 variable "ipam" {
-  type        = "string"
+  type        = string
   description = "The IPAM server to use for IP management."
   default     = ""
 }
 
 variable "ipam_token" {
-  type        = "string"
+  type        = string
   description = "The IPAM token to use for requests."
   default     = ""
 }
 
+//////
+// vSphere variables
+//////
+
+variable "vsphere_server" {
+  type        = string
+  description = "This is the vSphere server for the environment."
+}
+
+variable "vsphere_user" {
+  type        = string
+  description = "vSphere server user for the environment."
+}
+
+variable "vsphere_password" {
+  type        = string
+  description = "vSphere server password"
+}
+
+variable "vsphere_cluster" {
+  type        = string
+  description = "This is the name of the vSphere cluster."
+}
+
+variable "vsphere_datacenter" {
+  type        = string
+  description = "This is the name of the vSphere data center."
+}
+
+variable "vsphere_datastore" {
+  type        = string
+  description = "This is the name of the vSphere data store."
+}
+
+variable "vm_template" {
+  type        = string
+  description = "This is the name of the VM template to clone."
+}
+
+variable "vm_network" {
+  type        = string
+  description = "This is the name of the publicly accessible network for cluster ingress and access."
+  default     = "VM Network"
+}
+
+variable "vm_dns_addresses" {
+  type    = list(string)
+  default = ["1.1.1.1", "9.9.9.9"]
+}
+
 /////////
 // OpenShift cluster variables
 /////////
 
 variable "cluster_id" {
-  type        = "string"
+  type        = string
   description = "This cluster id must be of max length 27 and must have only alphanumeric or hyphen characters."
 }
 
 variable "base_domain" {
-  type        = "string"
+  type        = string
   description = "The base DNS zone to add the sub zone to."
 }
 
 variable "cluster_domain" {
-  type        = "string"
+  type        = string
   description = "The base DNS zone to add the sub zone to."
 }
 
 variable "machine_cidr" {
-  type = "string"
+  type = string
 }
 
 /////////
 // Bootstrap machine variables
 /////////
 
+variable "bootstrap_ignition_path" {
+  type    = string
+  default = "./bootstrap.ign"
+}
+
 variable "bootstrap_complete" {
-  type    = "string"
+  type    = string
   default = "false"
 }
 
-variable "bootstrap_ignition_url" {
-  type = "string"
+variable "bootstrap_ip_address" {
+  type    = string
+  default = ""
 }
 
-variable "bootstrap_ip" {
-  type    = "string"
+variable "lb_ip_address" {
+  type    = string
   default = ""
 }
 
 ///////////
-// Control Plane machine variables
+// control-plane machine variables
 ///////////
 
+variable "control_plane_ignition_path" {
+  type    = string
+  default = "./master.ign"
+}
+
 variable "control_plane_count" {
-  type    = "string"
+  type    = string
   default = "3"
 }
 
-variable "control_plane_ignition" {
-  type = "string"
-}
-
-variable "control_plane_ips" {
-  type    = "list"
+variable "control_plane_ip_addresses" {
+  type    = list(string)
   default = []
 }
-
-variable "master_memory" {
-  type = "string"
+variable "control_plane_memory" {
+  type    = string
   default = "16384"
 }
 
-variable "master_num_cpus" {
-  type = "string"
+variable "control_plane_num_cpus" {
+  type    = string
   default = "4"
 }
 
 //////////
-// Compute machine variables
+// compute machine variables
 //////////
 
+variable "compute_ignition_path" {
+  type    = string
+  default = "./worker.ign"
+}
+
 variable "compute_count" {
-  type    = "string"
+  type    = string
   default = "3"
 }
 
-variable "compute_ignition" {
-  type = "string"
-}
-
-variable "compute_ips" {
-  type    = "list"
+variable "compute_ip_addresses" {
+  type    = list(string)
   default = []
 }
 
 variable "compute_memory" {
-  type = "string"
+  type    = string
   default = "8192"
 }
 
 variable "compute_num_cpus" {
-  type = "string"
+  type    = string
   default = "4"
 }
+
+variable "ssh_public_key_path" {
+  type    = string
+  default = "~/.ssh/id_rsa.pub"
+}
diff --git a/upi/vsphere/vm/ifcfg.tmpl b/upi/vsphere/vm/ifcfg.tmpl
new file mode 100644
index 0000000000..84b12f4416
--- /dev/null
+++ b/upi/vsphere/vm/ifcfg.tmpl
@@ -0,0 +1,12 @@
+TYPE=Ethernet
+BOOTPROTO=none
+NAME=ens192
+DEVICE=ens192
+ONBOOT=yes
+IPADDR=${ip_address}
+PREFIX=${element(split("/", machine_cidr), 1)}
+GATEWAY=${cidrhost(machine_cidr, 1)}
+DOMAIN=${cluster_domain}
+%{ for index, ip in dns_addresses ~}
+DNS${index+1}=${ip}
+%{ endfor ~}
diff --git a/upi/vsphere/vm/ignition.tf b/upi/vsphere/vm/ignition.tf
new file mode 100644
index 0000000000..64ecd05426
--- /dev/null
+++ b/upi/vsphere/vm/ignition.tf
@@ -0,0 +1,47 @@
+locals {
+  ignition_encoded = "data:text/plain;charset=utf-8;base64,${base64encode(var.ignition)}"
+}
+
+data "ignition_file" "hostname" {
+  for_each = var.hostnames_ip_addresses
+
+  filesystem = "root"
+  path       = "/etc/hostname"
+  mode       = "420"
+
+  content {
+    content = element(split(".", each.key), 0)
+  }
+}
+
+data "ignition_file" "static_ip" {
+  for_each = var.hostnames_ip_addresses
+
+  filesystem = "root"
+  path       = "/etc/sysconfig/network-scripts/ifcfg-ens192"
+  mode       = "420"
+
+  content {
+    content = templatefile("${path.module}/ifcfg.tmpl", {
+      dns_addresses = var.dns_addresses,
+      machine_cidr  = var.machine_cidr
+      //ip_address     = var.hostnames_ip_addresses[count.index].value
+      ip_address     = each.value
+      cluster_domain = var.cluster_domain
+    })
+  }
+}
+
+data "ignition_config" "ign" {
+  for_each = var.hostnames_ip_addresses
+
+  append {
+    source = local.ignition_encoded
+  }
+
+  files = [
+    data.ignition_file.hostname[each.key].rendered,
+    data.ignition_file.static_ip[each.key].rendered,
+  ]
+}
+
diff --git a/upi/vsphere/vm/main.tf b/upi/vsphere/vm/main.tf
new file mode 100644
index 0000000000..3ae5e4d620
--- /dev/null
+++ b/upi/vsphere/vm/main.tf
@@ -0,0 +1,36 @@
+resource "vsphere_virtual_machine" "vm" {
+  for_each = var.hostnames_ip_addresses
+
+  name = element(split(".", each.key), 0)
+
+  resource_pool_id = var.resource_pool_id
+  datastore_id     = var.datastore_id
+  num_cpus         = var.num_cpus
+  memory           = var.memory
+  guest_id         = var.guest_id
+  folder           = var.folder_id
+  enable_disk_uuid = "true"
+
+  wait_for_guest_net_timeout  = "0"
+  wait_for_guest_net_routable = "false"
+
+  network_interface {
+    network_id = var.network_id
+  }
+
+  disk {
+    label            = "disk0"
+    size             = 60
+    thin_provisioned = var.disk_thin_provisioned
+  }
+
+  clone {
+    template_uuid = var.template_uuid
+  }
+
+  extra_config = {
+    "guestinfo.ignition.config.data"          = base64encode(data.ignition_config.ign[each.key].rendered)
+    "guestinfo.ignition.config.data.encoding" = "base64"
+  }
+}
+
diff --git a/upi/vsphere/vm/variables.tf b/upi/vsphere/vm/variables.tf
new file mode 100644
index 0000000000..41978df078
--- /dev/null
+++ b/upi/vsphere/vm/variables.tf
@@ -0,0 +1,61 @@
+variable "hostnames_ip_addresses" {
+  type = map(string)
+}
+
+variable "ignition" {
+  type    = string
+  default = ""
+}
+
+variable "disk_thin_provisioned" {
+  type = bool
+}
+
+variable "template_uuid" {
+  type = string
+}
+
+variable "guest_id" {
+  type = string
+}
+
+variable "resource_pool_id" {
+  type = string
+}
+
+variable "folder_id" {
+  type = string
+}
+
+variable "datastore_id" {
+  type = string
+}
+
+variable "network_id" {
+  type = string
+}
+
+variable "cluster_domain" {
+  type = string
+}
+
+variable "datacenter_id" {
+  type = string
+}
+
+variable "machine_cidr" {
+  type = string
+}
+
+variable "memory" {
+  type = string
+}
+
+variable "num_cpus" {
+  type = string
+}
+
+variable "dns_addresses" {
+  type = list(string)
+}
+
diff --git a/upi/vsphere/vm/versions.tf b/upi/vsphere/vm/versions.tf
new file mode 100644
index 0000000000..d9b6f790b9
--- /dev/null
+++ b/upi/vsphere/vm/versions.tf
@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}