diff --git a/.gitignore b/.gitignore index 828e4f5b87..de8604fab5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ assets*.zip **/ssh-key.pem build/** config.tfvars +generated/ diff --git a/common/bootkube/assets.tf b/common/bootkube/assets.tf new file mode 100644 index 0000000000..28b3c3b13d --- /dev/null +++ b/common/bootkube/assets.tf @@ -0,0 +1,62 @@ +# Self-hosted manifests (resources/generated/manifests/) +resource "template_folder" "bootkube" { + input_path = "${path.module}/resources/manifests" + output_path = "${path.cwd}/generated/manifests" + + vars { + hyperkube_image = "${var.container_images["hyperkube"]}" + pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}" + + etcd_servers = "${join(",", var.etcd_servers)}" + cloud_provider = "${var.cloud_provider}" + + cluster_cidr = "${var.cluster_cidr}" + service_cidr = "${var.service_cidr}" + kube_dns_service_ip = "${var.kube_dns_service_ip}" + advertise_address = "${var.advertise_address}" + + anonymous_auth = "${var.anonymous_auth}" + oidc_issuer_url = "${var.oidc_issuer_url}" + oidc_client_id = "${var.oidc_client_id}" + oidc_username_claim = "${var.oidc_username_claim}" + oidc_groups_claim = "${var.oidc_groups_claim}" + + ca_cert = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}" + apiserver_key = "${base64encode(tls_private_key.apiserver.private_key_pem)}" + apiserver_cert = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}" + serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}" + serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}" + } +} + +# kubeconfig (resources/generated/kubeconfig) +data "template_file" "kubeconfig" { + template = "${file("${path.module}/resources/kubeconfig")}" + + vars { + ca_cert = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}" + kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}" + kubelet_key = "${base64encode(tls_private_key.kubelet.private_key_pem)}" + server = "${var.kube_apiserver_url}" + } +} + +resource "localfile_file" "kubeconfig" { + content = "${data.template_file.kubeconfig.rendered}" + destination = "${path.cwd}/generated/kubeconfig" +} + +# bootkube.sh (resources/generated/bootkube.sh) +data "template_file" "bootkube" { + template = "${file("${path.module}/resources/bootkube.sh")}" + + vars { + bootkube_image = "${var.container_images["bootkube"]}" + etcd_server = "${element(var.etcd_servers, 0)}" + } +} + +resource "localfile_file" "bootkube" { + content = "${data.template_file.bootkube.rendered}" + destination = "${path.cwd}/generated/bootkube.sh" +} \ No newline at end of file diff --git a/common/bootkube/assets_tls.tf b/common/bootkube/assets_tls.tf new file mode 100644 index 0000000000..da9ce6d3b8 --- /dev/null +++ b/common/bootkube/assets_tls.tf @@ -0,0 +1,149 @@ +# Kubernetes CA (resources/generated/tls/{ca.crt,ca.key}) +resource "tls_private_key" "kube-ca" { + count = "${var.ca_cert == "" ? 1 : 0}" + + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_self_signed_cert" "kube-ca" { + count = "${var.ca_cert == "" ? 1 : 0}" + + key_algorithm = "${tls_private_key.kube-ca.algorithm}" + private_key_pem = "${tls_private_key.kube-ca.private_key_pem}" + + subject { + common_name = "kube-ca" + organization = "bootkube" + } + + is_ca_certificate = true + validity_period_hours = 8760 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "cert_signing", + ] +} + +resource "localfile_file" "kube-ca-key" { + content = "${var.ca_cert == "" ? tls_private_key.kube-ca.private_key_pem : var.ca_key}" + destination = "${path.cwd}/generated/tls/ca.key" +} + +resource "localfile_file" "kube-ca-crt" { + content = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.cert_pem : var.ca_cert}" + destination = "${path.cwd}/generated/tls/ca.crt" +} + +# Kubernetes API Server (resources/generated/tls/{apiserver.key,apiserver.crt}) +resource "tls_private_key" "apiserver" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_cert_request" "apiserver" { + key_algorithm = "${tls_private_key.apiserver.algorithm}" + private_key_pem = "${tls_private_key.apiserver.private_key_pem}" + + subject { + common_name = "kube-apiserver" + organization = "kube-master" + } + + dns_names = [ + "${replace(element(split(":", var.kube_apiserver_url), 1), "/", "")}", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + ] + + ip_addresses = [ + "${var.kube_apiserver_service_ip}", + ] +} + +resource "tls_locally_signed_cert" "apiserver" { + cert_request_pem = "${tls_cert_request.apiserver.cert_request_pem}" + + ca_key_algorithm = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.key_algorithm : var.ca_key_alg}" + ca_private_key_pem = "${var.ca_cert == "" ? tls_private_key.kube-ca.private_key_pem : var.ca_key}" + ca_cert_pem = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.cert_pem : var.ca_cert}" + + validity_period_hours = 8760 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + "client_auth", + ] +} + +resource "localfile_file" "apiserver-key" { + content = "${tls_private_key.apiserver.private_key_pem}" + destination = "${path.cwd}/generated/tls/apiserver.key" +} + +resource "localfile_file" "apiserver-crt" { + content = "${tls_locally_signed_cert.apiserver.cert_pem}" + destination = "${path.cwd}/generated/tls/apiserver.crt" +} + +# Kubernete's Service Account (resources/generated/tls/{service-account.key,service-account.pub}) +resource "tls_private_key" "service-account" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "localfile_file" "service-account-key" { + content = "${tls_private_key.service-account.private_key_pem}" + destination = "${path.cwd}/generated/tls/service-account.key" +} + +resource "localfile_file" "service-account-crt" { + content = "${tls_private_key.service-account.public_key_pem}" + destination = "${path.cwd}/generated/tls/service-account.pub" +} + +# Kubelet +resource "tls_private_key" "kubelet" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_cert_request" "kubelet" { + key_algorithm = "${tls_private_key.kubelet.algorithm}" + private_key_pem = "${tls_private_key.kubelet.private_key_pem}" + + subject { + common_name = "kubelet" + organization = "system:masters" + } +} + +resource "tls_locally_signed_cert" "kubelet" { + cert_request_pem = "${tls_cert_request.kubelet.cert_request_pem}" + + ca_key_algorithm = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.key_algorithm : var.ca_key_alg}" + ca_private_key_pem = "${var.ca_cert == "" ? tls_private_key.kube-ca.private_key_pem : var.ca_key}" + ca_cert_pem = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.cert_pem : var.ca_cert}" + + validity_period_hours = 8760 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + "client_auth", + ] +} + +resource "localfile_file" "kubelet-key" { + content = "${tls_private_key.kubelet.private_key_pem}" + destination = "${path.cwd}/generated/tls/kubelet.key" +} + +resource "localfile_file" "kubelet-crt" { + content = "${tls_locally_signed_cert.kubelet.cert_pem}" + destination = "${path.cwd}/generated/tls/kubelet.crt" +} diff --git a/common/bootkube/outputs.tf b/common/bootkube/outputs.tf new file mode 100644 index 0000000000..5714dc0100 --- /dev/null +++ b/common/bootkube/outputs.tf @@ -0,0 +1,15 @@ +output "kubeconfig" { + value = "${data.template_file.kubeconfig.rendered}" +} + +output "ca_cert" { + value = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.cert_pem : var.ca_cert}" +} + +output "ca_key_alg" { + value = "${var.ca_cert == "" ? tls_self_signed_cert.kube-ca.key_algorithm : var.ca_key_alg}" +} + +output "ca_key" { + value = "${var.ca_cert == "" ? tls_private_key.kube-ca.private_key_pem : var.ca_key}" +} diff --git a/common/bootkube/resources/bootkube.sh b/common/bootkube/resources/bootkube.sh new file mode 100644 index 0000000000..5a05da8b27 --- /dev/null +++ b/common/bootkube/resources/bootkube.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +/usr/bin/rkt run \ + --trust-keys-from-https \ + --volume assets,kind=host,source=$(pwd) \ + --mount volume=assets,target=/assets \ + ${bootkube_image} \ + --net=host \ + --dns=host \ + --exec=/bootkube -- start --asset-dir=/assets --etcd-server=${etcd_server} \ No newline at end of file diff --git a/common/bootkube/resources/kubeconfig b/common/bootkube/resources/kubeconfig new file mode 100644 index 0000000000..5d8fb0c3cb --- /dev/null +++ b/common/bootkube/resources/kubeconfig @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: ${server} + certificate-authority-data: ${ca_cert} +users: +- name: kubelet + user: + client-certificate-data: ${kubelet_cert} + client-key-data: ${kubelet_key} +contexts: +- context: + cluster: local + user: kubelet diff --git a/common/bootkube/resources/manifests/kube-apiserver-secret.yaml b/common/bootkube/resources/manifests/kube-apiserver-secret.yaml new file mode 100644 index 0000000000..583ec591d1 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-apiserver-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kube-apiserver + namespace: kube-system +type: Opaque +data: + apiserver.key: ${apiserver_key} + apiserver.crt: ${apiserver_cert} + service-account.pub: ${serviceaccount_pub} + ca.crt: ${ca_cert} \ No newline at end of file diff --git a/common/bootkube/resources/manifests/kube-apiserver.yaml b/common/bootkube/resources/manifests/kube-apiserver.yaml new file mode 100644 index 0000000000..f746add0ff --- /dev/null +++ b/common/bootkube/resources/manifests/kube-apiserver.yaml @@ -0,0 +1,76 @@ +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: kube-apiserver + namespace: kube-system + labels: + k8s-app: kube-apiserver +spec: + template: + metadata: + labels: + k8s-app: kube-apiserver + annotations: + checkpointer.alpha.coreos.com/checkpoint: "true" + spec: + nodeSelector: + master: "true" + hostNetwork: true + containers: + - name: kube-apiserver + image: ${hyperkube_image} + command: + - /usr/bin/flock + - --exclusive + - --timeout=30 + - /var/lock/api-server.lock + - /hyperkube + - apiserver + - --bind-address=0.0.0.0 + - --secure-port=443 + - --insecure-port=8080 + - --advertise-address=${advertise_address} + - --etcd-servers=${etcd_servers} + - --storage-backend=etcd3 + - --allow-privileged=true + - --service-cluster-ip-range=${service_cidr} + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota + - --runtime-config=api/all=true + - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt + - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key + - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub + - --client-ca-file=/etc/kubernetes/secrets/ca.crt + - --authorization-mode=RBAC + - --runtime-config=rbac.authorization.k8s.io/v1alpha1 + - --anonymous-auth=${anonymous_auth} + - --oidc-issuer-url=${oidc_issuer_url} + - --oidc-client-id=${oidc_client_id} + - --oidc-username-claim=${oidc_username_claim} + - --oidc-groups-claim=${oidc_groups_claim} + - --oidc-ca-file=/etc/kubernetes/secrets/ca.crt + - --cloud-provider=${cloud_provider} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /etc/kubernetes/secrets + name: secrets + readOnly: true + - mountPath: /var/lock + name: var-lock + readOnly: false + volumes: + - name: ssl-certs-host + hostPath: + path: /usr/share/ca-certificates + - name: secrets + secret: + secretName: kube-apiserver + - name: var-lock + hostPath: + path: /var/lock \ No newline at end of file diff --git a/common/bootkube/resources/manifests/kube-controller-manager-disruption.yaml b/common/bootkube/resources/manifests/kube-controller-manager-disruption.yaml new file mode 100644 index 0000000000..d7f85505fb --- /dev/null +++ b/common/bootkube/resources/manifests/kube-controller-manager-disruption.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: kube-controller-manager + namespace: kube-system +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: kube-controller-manager \ No newline at end of file diff --git a/common/bootkube/resources/manifests/kube-controller-manager-secret.yaml b/common/bootkube/resources/manifests/kube-controller-manager-secret.yaml new file mode 100644 index 0000000000..50efc76a0d --- /dev/null +++ b/common/bootkube/resources/manifests/kube-controller-manager-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kube-controller-manager + namespace: kube-system +type: Opaque +data: + service-account.key: ${serviceaccount_key} + ca.crt: ${ca_cert} \ No newline at end of file diff --git a/common/bootkube/resources/manifests/kube-controller-manager.yaml b/common/bootkube/resources/manifests/kube-controller-manager.yaml new file mode 100644 index 0000000000..941bad7a75 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-controller-manager.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-controller-manager + namespace: kube-system + labels: + k8s-app: kube-controller-manager +spec: + replicas: 2 + template: + metadata: + labels: + k8s-app: kube-controller-manager + spec: + nodeSelector: + master: "true" + containers: + - name: kube-controller-manager + image: ${hyperkube_image} + command: + - ./hyperkube + - controller-manager + - --allocate-node-cidrs=true + - --configure-cloud-routes=false + - --cluster-cidr=${cluster_cidr} + - --root-ca-file=/etc/kubernetes/secrets/ca.crt + - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key + - --leader-elect=true + - --cloud-provider=${cloud_provider} + volumeMounts: + - name: secrets + mountPath: /etc/kubernetes/secrets + readOnly: true + - name: ssl-host + mountPath: /etc/ssl/certs + readOnly: true + volumes: + - name: secrets + secret: + secretName: kube-controller-manager + - name: ssl-host + hostPath: + path: /usr/share/ca-certificates + dnsPolicy: Default # Don't use cluster DNS. diff --git a/common/bootkube/resources/manifests/kube-dns.yaml b/common/bootkube/resources/manifests/kube-dns.yaml new file mode 100644 index 0000000000..1ade334dd0 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-dns.yaml @@ -0,0 +1,172 @@ +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "KubeDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: ${kube_dns_service_ip} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: kubedns + image: ${kubedns_image} + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz-kubedns + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain=cluster.local. + - --dns-port=10053 + - --config-map=kube-dns + # This should be set to v=2 only after the new image (cut from 1.5) has + # been released, otherwise we will flood the logs. + - --v=0 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + - name: dnsmasq + image: ${kubednsmasq_image} + livenessProbe: + httpGet: + path: /healthz-dnsmasq + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + - --log-facility=- + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 10Mi + - name: dnsmasq-metrics + image: ${dnsmasq_metrics_image} + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 10Mi + - name: healthz + image: ${exechealthz_image} + resources: + limits: + memory: 50Mi + requests: + cpu: 10m + # Note that this container shouldn't really need 50Mi of memory. The + # limits are set higher than expected pending investigation on #29688. + # The extra memory was stolen from the kubedns container to keep the + # net memory requested by the pod constant. + memory: 50Mi + args: + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1 >/dev/null + - --url=/healthz-dnsmasq + - --cmd=nslookup kubernetes.default.svc.cluster.local 127.0.0.1:10053 >/dev/null + - --url=/healthz-kubedns + - --port=8080 + - --quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default # Don't use cluster DNS. diff --git a/common/bootkube/resources/manifests/kube-flannel.yaml b/common/bootkube/resources/manifests/kube-flannel.yaml new file mode 100644 index 0000000000..47763600e8 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-flannel.yaml @@ -0,0 +1,85 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-flannel-cfg + namespace: kube-system + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } + } + net-conf.json: | + { + "Network": "${cluster_cidr}", + "Backend": { + "Type": "vxlan" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel + namespace: kube-system + labels: + tier: node + app: flannel +spec: + template: + metadata: + labels: + tier: node + app: flannel + spec: + hostNetwork: true + containers: + - name: kube-flannel + image: ${flannel_image} + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr", "--iface=$(POD_IP)"] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: run + mountPath: /run + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: install-cni + image: busybox + command: [ "/bin/sh", "-c", "set -e -x; TMP=/etc/cni/net.d/.tmp-flannel-cfg; cp /etc/kube-flannel/cni-conf.json $TMP; mv $TMP /etc/cni/net.d/10-flannel.conf; while :; do sleep 3600; done" ] + volumeMounts: + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/kubernetes/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg diff --git a/common/bootkube/resources/manifests/kube-proxy.yaml b/common/bootkube/resources/manifests/kube-proxy.yaml new file mode 100644 index 0000000000..1279f17959 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-proxy.yaml @@ -0,0 +1,45 @@ +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: kube-proxy + namespace: kube-system + labels: + k8s-app: kube-proxy +spec: + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + hostNetwork: true + containers: + - name: kube-proxy + image: ${hyperkube_image} + command: + - /hyperkube + - proxy + - --kubeconfig=/etc/kubernetes/kubeconfig + - --proxy-mode=iptables + - --hostname-override=$(NODE_NAME) + - --cluster-cidr=${cluster_cidr} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - name: etc-kubernetes + mountPath: /etc/kubernetes + readOnly: true + volumes: + - hostPath: + path: /usr/share/ca-certificates + name: ssl-certs-host + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes diff --git a/common/bootkube/resources/manifests/kube-scheduler-disruption.yaml b/common/bootkube/resources/manifests/kube-scheduler-disruption.yaml new file mode 100644 index 0000000000..459b08a4a6 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-scheduler-disruption.yaml @@ -0,0 +1,10 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: kube-scheduler + namespace: kube-system +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: kube-scheduler diff --git a/common/bootkube/resources/manifests/kube-scheduler.yaml b/common/bootkube/resources/manifests/kube-scheduler.yaml new file mode 100644 index 0000000000..36fb078f92 --- /dev/null +++ b/common/bootkube/resources/manifests/kube-scheduler.yaml @@ -0,0 +1,23 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-scheduler + namespace: kube-system + labels: + k8s-app: kube-scheduler +spec: + replicas: 2 + template: + metadata: + labels: + k8s-app: kube-scheduler + spec: + nodeSelector: + master: "true" + containers: + - name: kube-scheduler + image: ${hyperkube_image} + command: + - ./hyperkube + - scheduler + - --leader-elect=true diff --git a/common/bootkube/resources/manifests/kube-system-rbac-role-binding.yaml b/common/bootkube/resources/manifests/kube-system-rbac-role-binding.yaml new file mode 100644 index 0000000000..59cb24c80f --- /dev/null +++ b/common/bootkube/resources/manifests/kube-system-rbac-role-binding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: system:default-sa +subjects: + - kind: ServiceAccount + name: default + namespace: kube-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/common/bootkube/resources/manifests/pod-checkpoint-installer.yaml b/common/bootkube/resources/manifests/pod-checkpoint-installer.yaml new file mode 100644 index 0000000000..b18eb325fb --- /dev/null +++ b/common/bootkube/resources/manifests/pod-checkpoint-installer.yaml @@ -0,0 +1,28 @@ +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: checkpoint-installer + namespace: kube-system + labels: + k8s-app: pod-checkpoint-installer +spec: + template: + metadata: + labels: + k8s-app: pod-checkpoint-installer + spec: + nodeSelector: + master: "true" + hostNetwork: true + containers: + - name: checkpoint-installer + image: ${pod_checkpointer_image} + command: + - /checkpoint-installer.sh + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: etc-k8s-manifests + volumes: + - name: etc-k8s-manifests + hostPath: + path: /etc/kubernetes/manifests diff --git a/common/bootkube/variables.tf b/common/bootkube/variables.tf new file mode 100644 index 0000000000..02a1d0e5a8 --- /dev/null +++ b/common/bootkube/variables.tf @@ -0,0 +1,84 @@ +variable "container_images" { + description = "Container images to use" + type = "map" +} + +variable "kube_apiserver_url" { + description = "URL used to reach kube-apiserver" + type = "string" +} + +variable "kube_apiserver_service_ip" { + description = "Service IP used to reach kube-apiserver inside the cluster" + type = "string" +} + +variable "kube_dns_service_ip" { + description = "Service IP used to reach kube-dns" + type = "string" +} + +variable "etcd_servers" { + description = "List of etcd servers to connect with (scheme://ip:port)" + type = "list" +} + +variable "cloud_provider" { + description = "The provider for cloud services (empty string for no provider)" + type = "string" +} + +variable "service_cidr" { + description = "A CIDR notation IP range from which to assign service cluster IPs" + type = "string" +} + +variable "cluster_cidr" { + description = "A CIDR notation IP range from which to assign pod IPs" + type = "string" +} + +variable "advertise_address" { + description = "The IP address on which to advertise the apiserver to members of the cluster" + type = "string" +} + +variable "ca_cert" { + description = "PEM-encoded CA certificate (generated if blank)" + type = "string" +} + +variable "ca_key_alg" { + description = "Algorithm used to generate ca_key (required if ca_cert is specified)" + type = "string" +} + +variable "ca_key" { + description = "PEM-encoded CA key (required if ca_cert is specified)" + type = "string" +} + +variable "anonymous_auth" { + description = "Enables anonymous requests to the secure port of the API server" + type = "string" +} + +variable "oidc_issuer_url" { + description = "The URL of the OpenID issuer, only HTTPS scheme will be accepted" + type = "string" +} + +variable "oidc_client_id" { + description = "The client ID for the OpenID Connect client" + type = "string" +} + +variable "oidc_username_claim" { + description = "The OpenID claim to use as the user name" + type = "string" +} + +variable "oidc_groups_claim" { + description = "The OpenID claim to use for specifying user groups (string or array of strings)" + type = "string" +} diff --git a/common/tectonic/assets.tf b/common/tectonic/assets.tf new file mode 100644 index 0000000000..67cdbe246b --- /dev/null +++ b/common/tectonic/assets.tf @@ -0,0 +1,77 @@ +# Kubernetes Manifests (resources/generated/manifests/) +## github.com/coreos-inc/tectonic/commit/0b48144d5332201cf461a309d501b33a00a26f75 +resource "template_folder" "tectonic" { + input_path = "${path.module}/resources/manifests" + output_path = "${path.cwd}/generated/tectonic" + + vars { + console_image = "${var.container_images["console"]}" + identity_image = "${var.container_images["identity"]}" + kube_version_operator_image = "${var.container_images["kube_version_operator"]}" + tectonic_channel_operator_image = "${var.container_images["tectonic_channel_operator"]}" + node_agent_image = "${var.container_images["node_agent"]}" + prometheus_operator_image = "${var.container_images["prometheus_operator"]}" + node_exporter_image = "${var.container_images["node_exporter"]}" + config_reload_image = "${var.container_images["config_reload"]}" + heapster_image = "${var.container_images["heapster"]}" + addon_resizer_image = "${var.container_images["addon_resizer"]}" + stats_emitter_image = "${var.container_images["stats_emitter"]}" + stats_extender_image = "${var.container_images["stats_extender"]}" + error_server_image = "${var.container_images["error_server"]}" + ingress_controller_image = "${var.container_images["ingress_controller"]}" + + prometheus_version = "${var.versions["prometheus"]}" + kubernetes_version = "${var.versions["kubernetes"]}" + tectonic_version = "${var.versions["tectonic"]}" + + license = "${base64encode(var.license)}" + pull_secret = "${base64encode(var.pull_secret)}" + ca_cert = "${base64encode(var.ca_cert)}" + + update_server = "${var.update_server}" + update_channel = "${var.update_channel}" + update_app_id = "${var.update_app_id}" + + admin_user_id = "${random_id.admin_user_id.b64}" + admin_email = "${var.admin_email}" + admin_password_hash = "${var.admin_password_hash}" + + console_base_address = "https://${var.domain}" + console_client_id = "${var.console_client_id}" + console_secret = "${random_id.console_secret.b64}" + console_callback = "https://${var.domain}/auth/callback" + + ingress_kind = "${var.ingress_kind}" + ingress_tls_cert = "${base64encode(tls_locally_signed_cert.ingress.cert_pem)}" + ingress_tls_key = "${base64encode(tls_private_key.ingress.private_key_pem)}" + + identity_server_tls_cert = "${base64encode(tls_locally_signed_cert.identity-server.cert_pem)}" + identity_server_tls_key = "${base64encode(tls_private_key.identity-server.private_key_pem)}" + identity_client_tls_cert = "${base64encode(tls_locally_signed_cert.identity-client.cert_pem)}" + identity_client_tls_key = "${base64encode(tls_private_key.identity-client.private_key_pem)}" + + kubectl_client_id = "${var.kubectl_client_id}" + kubectl_secret = "${random_id.kubectl_secret.b64}" + + kube_apiserver_url = "${var.kube_apiserver_url}" + oidc_issuer_url = "https://${var.domain}/identity" + + cluster_id = "${uuid()}" + platform = "${var.platform}" + certificates_strategy = "${var.ca_generated == "true" ? "installerGeneratedCA" : "userProvidedCA"}" + } +} + +# tectonic.sh (resources/generated/tectonic.sh) +data "template_file" "tectonic" { + template = "${file("${path.module}/resources/tectonic.sh")}" + + vars { + ingress_kind = "${var.ingress_kind}" + } +} + +resource "localfile_file" "tectonic" { + content = "${data.template_file.tectonic.rendered}" + destination = "${path.cwd}/generated/tectonic.sh" +} \ No newline at end of file diff --git a/common/tectonic/crypto.tf b/common/tectonic/crypto.tf new file mode 100644 index 0000000000..79f18312c2 --- /dev/null +++ b/common/tectonic/crypto.tf @@ -0,0 +1,101 @@ +# Cryptographically-secure ramdon strings used by various components. + +resource "random_id" "admin_user_id" { + byte_length = 16 +} + +resource "random_id" "kubectl_secret" { + byte_length = 16 +} + +resource "random_id" "console_secret" { + byte_length = 16 +} + +# Ingress' server certificate + +resource "tls_private_key" "ingress" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_cert_request" "ingress" { + key_algorithm = "${tls_private_key.ingress.algorithm}" + private_key_pem = "${tls_private_key.ingress.private_key_pem}" + + subject { + common_name = "${var.domain}" + } +} + +resource "tls_locally_signed_cert" "ingress" { + cert_request_pem = "${tls_cert_request.ingress.cert_request_pem}" + + ca_key_algorithm = "${var.ca_key_alg}" + ca_private_key_pem = "${var.ca_key}" + ca_cert_pem = "${var.ca_cert}" + + validity_period_hours = 8760 + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + "client_auth", + ] +} + +# Identity's gRPC server/client certificates + +resource "tls_private_key" "identity-server" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_cert_request" "identity-server" { + key_algorithm = "${tls_private_key.identity-server.algorithm}" + private_key_pem = "${tls_private_key.identity-server.private_key_pem}" + + subject { + common_name = "tectonic-identity-api.tectonic-system.svc.cluster.local" + } +} + +resource "tls_locally_signed_cert" "identity-server" { + cert_request_pem = "${tls_cert_request.identity-server.cert_request_pem}" + + ca_key_algorithm = "${var.ca_key_alg}" + ca_private_key_pem = "${var.ca_key}" + ca_cert_pem = "${var.ca_cert}" + + validity_period_hours = 8760 + allowed_uses = [ + "server_auth", + ] +} + +resource "tls_private_key" "identity-client" { + algorithm = "RSA" + rsa_bits = "2048" +} + +resource "tls_cert_request" "identity-client" { + key_algorithm = "${tls_private_key.identity-client.algorithm}" + private_key_pem = "${tls_private_key.identity-client.private_key_pem}" + + subject { + common_name = "tectonic-identity-api.tectonic-system.svc.cluster.local" + } +} + +resource "tls_locally_signed_cert" "identity-client" { + cert_request_pem = "${tls_cert_request.identity-client.cert_request_pem}" + + ca_key_algorithm = "${var.ca_key_alg}" + ca_private_key_pem = "${var.ca_key}" + ca_cert_pem = "${var.ca_cert}" + + validity_period_hours = 8760 + allowed_uses = [ + "client_auth", + ] +} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/config.yaml b/common/tectonic/resources/manifests/config.yaml new file mode 100644 index 0000000000..cecb75b35f --- /dev/null +++ b/common/tectonic/resources/manifests/config.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: tectonic-config + namespace: tectonic-system +data: + clusterID: "${cluster_id}" + installerPlatform: "${platform}" + certificatesStrategy: "${certificates_strategy}" + tectonicUpdaterEnabled: "true" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/console/deployment.yaml b/common/tectonic/resources/manifests/console/deployment.yaml new file mode 100644 index 0000000000..e7c81ca9f0 --- /dev/null +++ b/common/tectonic/resources/manifests/console/deployment.yaml @@ -0,0 +1,125 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: tectonic-console + component: ui + name: tectonic-console + namespace: tectonic-system +spec: + replicas: 2 + selector: + matchLabels: + app: tectonic-console + component: ui + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: tectonic-console + component: ui + name: tectonic-console + spec: + containers: + - command: + - /opt/bridge/bin/bridge + env: + - name: BRIDGE_K8S_MODE + value: in-cluster + - name: BRIDGE_K8S_AUTH + value: oidc + - name: BRIDGE_K8S_PUBLIC_ENDPOINT + value: ${kube_apiserver_url} + - name: BRIDGE_LISTEN + value: http://0.0.0.0:80 + - name: BRIDGE_BASE_ADDRESS + value: ${console_base_address} + - name: BRIDGE_BASE_PATH + value: / + - name: BRIDGE_PUBLIC_DIR + value: /opt/bridge/static + - name: BRIDGE_USER_AUTH + value: oidc + - name: BRIDGE_USER_AUTH_OIDC_ISSUER_URL + value: ${oidc_issuer_url} + - name: BRIDGE_USER_AUTH_OIDC_CLIENT_ID + value: ${console_client_id} + - name: BRIDGE_USER_AUTH_OIDC_CLIENT_SECRET + value: ${console_secret} + - name: BRIDGE_KUBECTL_CLIENT_ID + value: ${kubectl_client_id } + - name: BRIDGE_KUBECTL_CLIENT_SECRET + value: ${kubectl_secret} + - name: BRIDGE_TECTONIC_VERSION + value: ${tectonic_version} + - name: BRIDGE_CA_FILE + value: /etc/tectonic-ca-cert-secret/ca-cert + - name: BRIDGE_LICENSE_FILE + value: /etc/tectonic/licenses/license + image: ${console_image} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health + port: 80 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: tectonic-console + ports: + - containerPort: 80 + protocol: TCP + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 100m + memory: 50Mi + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /etc/tectonic-ca-cert-secret + name: tectonic-ca-cert-secret + readOnly: true + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /usr/share/ca-certificates + name: ca-certs-host + readOnly: true + - mountPath: /etc/tectonic/licenses + name: tectonic-license-secret + readOnly: true + - mountPath: /etc/tectonic-identity-grpc-client-secret + name: tectonic-identity-grpc-client-secret + readOnly: true + dnsPolicy: ClusterFirst + imagePullSecrets: + - name: coreos-pull-secret + restartPolicy: Always + securityContext: {} + terminationGracePeriodSeconds: 30 + volumes: + - name: tectonic-ca-cert-secret + secret: + secretName: tectonic-ca-cert-secret + - hostPath: + path: /etc/ssl/certs + name: ssl-certs-host + - hostPath: + path: /usr/share/ca-certificates + name: ca-certs-host + - name: tectonic-license-secret + secret: + secretName: tectonic-license-secret + - name: tectonic-identity-grpc-client-secret + secret: + secretName: tectonic-identity-grpc-client-secret diff --git a/common/tectonic/resources/manifests/console/service.yaml b/common/tectonic/resources/manifests/console/service.yaml new file mode 100644 index 0000000000..94cddb9fdf --- /dev/null +++ b/common/tectonic/resources/manifests/console/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: tectonic-console + labels: + app: tectonic-console + component: ui +spec: + selector: + app: tectonic-console + component: ui + type: NodePort + ports: + - name: tectonic-console + protocol: TCP + port: 80 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/heapster/deployment.yaml b/common/tectonic/resources/manifests/heapster/deployment.yaml new file mode 100644 index 0000000000..a7700344cf --- /dev/null +++ b/common/tectonic/resources/manifests/heapster/deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: heapster + namespace: kube-system + labels: + k8s-app: heapster + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: heapster + template: + metadata: + labels: + k8s-app: heapster + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: heapster + image: ${heapster_image} + command: + - /heapster + - --source=kubernetes.summary_api:'' + livenessProbe: + httpGet: + path: /healthz + port: 8082 + scheme: HTTP + initialDelaySeconds: 180 + timeoutSeconds: 5 + - name: heapster-nanny + image: ${addon_resizer_image} + command: + - /pod_nanny + - --cpu=80m + - --extra-cpu=0.5m + - --memory=140Mi + - --extra-memory=4Mi + - --threshold=5 + - --deployment=heapster + - --container=heapster + - --poll-period=300000 + - --estimator=exponential + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + limits: + cpu: 50m + memory: 90Mi + requests: + cpu: 50m + memory: 90Mi \ No newline at end of file diff --git a/common/tectonic/resources/manifests/heapster/service.yaml b/common/tectonic/resources/manifests/heapster/service.yaml new file mode 100644 index 0000000000..a6c79f23fb --- /dev/null +++ b/common/tectonic/resources/manifests/heapster/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: heapster + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Heapster" +spec: + selector: + k8s-app: heapster + ports: + - port: 80 + targetPort: 8082 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/identity/configmap.yaml b/common/tectonic/resources/manifests/identity/configmap.yaml new file mode 100644 index 0000000000..05e9445fc6 --- /dev/null +++ b/common/tectonic/resources/manifests/identity/configmap.yaml @@ -0,0 +1,42 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: tectonic-identity + namespace: tectonic-system +data: + config.yaml: | + issuer: ${oidc_issuer_url} + storage: + type: kubernetes + config: + inCluster: true + web: + http: 0.0.0.0:5556 + grpc: + addr: 0.0.0.0:5557 + tlsCert: /etc/tectonic-identity-grpc-server-secret/tls-cert + tlsKey: /etc/tectonic-identity-grpc-server-secret/tls-key + tlsClientCA: /etc/tectonic-identity-grpc-server-secret/ca-cert + frontend: + theme: 'tectonic' + issuer: 'Tectonic Identity' + oauth2: + skipApprovalScreen: true + staticClients: + - id: ${console_client_id} + redirectURIs: + - '${console_callback}' + name: 'Tectonic Console' + secret: ${console_secret} + - id: ${kubectl_client_id} + public: true + trustedPeers: + - ${console_client_id} + name: 'Kubectl' + secret: ${kubectl_secret} + enablePasswordDB: true + staticPasswords: + - email: "${admin_email}" + hash: "${admin_password_hash}" + username: "admin" + userID: "${admin_user_id}" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/identity/deployment.yaml b/common/tectonic/resources/manifests/identity/deployment.yaml new file mode 100644 index 0000000000..95797ad349 --- /dev/null +++ b/common/tectonic/resources/manifests/identity/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: tectonic-identity + namespace: tectonic-system + labels: + app: tectonic-identity + component: identity +spec: + replicas: 1 + # New identity pods must be healthy for 30 seconds + # before they're marked as ready. + minReadySeconds: 30 + strategy: + rollingUpdate: + # During a rolling update every deployed pod must be + # ready before the update terminates an existing pod. + maxUnavailable: 0 + template: + metadata: + name: tectonic-identity + labels: + app: tectonic-identity + component: identity + spec: + volumes: + - name: config + configMap: + name: tectonic-identity + items: + - key: config.yaml + path: config.yaml + - name: tectonic-identity-grpc-server-secret + secret: + secretName: tectonic-identity-grpc-server-secret + containers: + - name: tectonic-identity + imagePullPolicy: IfNotPresent + image: ${identity_image} + command: ["/usr/local/bin/dex", "serve", "/etc/dex/config.yaml"] + volumeMounts: + - name: config + mountPath: /etc/dex + - name: tectonic-identity-grpc-server-secret + mountPath: /etc/tectonic-identity-grpc-server-secret + readOnly: true + ports: + - containerPort: 5556 + protocol: TCP + - containerPort: 5557 + protocol: TCP + livenessProbe: + httpGet: + path: /identity/healthz + port: 5556 + initialDelaySeconds: 5 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 100m + memory: 50Mi + # For development, assume that all images we build ourselves are under a + # private registry. + imagePullSecrets: + - name: coreos-pull-secret \ No newline at end of file diff --git a/common/tectonic/resources/manifests/identity/services.yaml b/common/tectonic/resources/manifests/identity/services.yaml new file mode 100644 index 0000000000..11296e2a1b --- /dev/null +++ b/common/tectonic/resources/manifests/identity/services.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: tectonic-identity + namespace: tectonic-system + labels: + app: tectonic-identity + component: identity +spec: + selector: + app: tectonic-identity + component: identity + ports: + - name: worker + protocol: TCP + port: 5556 +--- +apiVersion: v1 +kind: Service +metadata: + name: tectonic-identity-api + namespace: tectonic-system +spec: + selector: + app: tectonic-identity + component: identity + ports: + - name: api + protocol: TCP + port: 5557 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/default-backend/configmap.yaml b/common/tectonic/resources/manifests/ingress/default-backend/configmap.yaml new file mode 100644 index 0000000000..0f885aac2e --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/default-backend/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tectonic-custom-error +data: + custom-http-errors: "400,401,403,404,500,503,504" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/default-backend/deployment.yaml b/common/tectonic/resources/manifests/ingress/default-backend/deployment.yaml new file mode 100644 index 0000000000..3fa43bcc16 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/default-backend/deployment.yaml @@ -0,0 +1,36 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend +spec: + replicas: 1 + template: + metadata: + labels: + app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: ${error_server_image} + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + imagePullSecrets: + - name: coreos-pull-secret \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/default-backend/service.yaml b/common/tectonic/resources/manifests/ingress/default-backend/service.yaml new file mode 100644 index 0000000000..df05d65b10 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/default-backend/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + labels: + app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: default-http-backend \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/hostport.yaml b/common/tectonic/resources/manifests/ingress/hostport.yaml new file mode 100644 index 0000000000..9ee3635921 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/hostport.yaml @@ -0,0 +1,77 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: tectonic-ingress-controller + namespace: tectonic-system + labels: + app: tectonic-lb + component: ingress-controller + type: nginx +spec: + template: + metadata: + labels: + app: tectonic-lb + component: ingress-controller + type: nginx + annotations: + scheduler.alpha.kubernetes.io/affinity: > + { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "master", + "operator": "DoesNotExist" + } + ] + } + ] + } + } + } + spec: + containers: + - name: nginx-ingress-lb + image: ${ingress_controller_image} + args: + - /nginx-ingress-controller + - --nginx-configmap=$(POD_NAMESPACE)/tectonic-custom-error + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --default-ssl-certificate=tectonic-system/tectonic-ingress-tls-secret + - --watch-namespace=tectonic-system + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + hostPort: 80 + - name: https + containerPort: 443 + hostPort: 443 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + hostNetwork: true + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 60 diff --git a/common/tectonic/resources/manifests/ingress/ingress.yaml b/common/tectonic/resources/manifests/ingress/ingress.yaml new file mode 100644 index 0000000000..3ce59841f1 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: tectonic-ingress + namespace: tectonic-system + annotations: + ingress.kubernetes.io/ssl-redirect: "true" +spec: + tls: + - secretName: tectonic-ingress-tls-secret + rules: + - http: + paths: + - path: / + backend: + serviceName: tectonic-console + servicePort: 80 + - path: /identity + backend: + serviceName: tectonic-identity + servicePort: 5556 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/nodeport/configmap.yaml b/common/tectonic/resources/manifests/ingress/nodeport/configmap.yaml new file mode 100644 index 0000000000..9431f46c14 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/nodeport/configmap.yaml @@ -0,0 +1,379 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tectonic-ingress-nginx-config +data: + # Taken from https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx.tmpl + # This makes one minor modification changing the "port_in_redirect" option to + # "on". This is required because we run Ingress on a node port instead of + # 80/443 and the node port must be included in the redirect for it to work. + nginx.tmpl: |- + {{ $cfg := .cfg }} + daemon off; + + worker_processes {{ $cfg.workerProcesses }}; + + pid /run/nginx.pid; + + worker_rlimit_nofile 131072; + + pcre_jit on; + + events { + multi_accept on; + worker_connections {{ $cfg.maxWorkerConnections }}; + use epoll; + } + + http { + {{/* we use the value of the header X-Forwarded-For to be able to use the geo_ip module */}} + {{ if $cfg.useProxyProtocol -}} + set_real_ip_from {{ $cfg.proxyRealIpCidr }}; + real_ip_header proxy_protocol; + {{ else }} + real_ip_header X-Forwarded-For; + set_real_ip_from 0.0.0.0/0; + {{ end -}} + + real_ip_recursive on; + + {{/* databases used to determine the country depending on the client IP address */}} + {{/* http://nginx.org/en/docs/http/ngx_http_geoip_module.html */}} + {{/* this is require to calculate traffic for individual country using GeoIP in the status page */}} + geoip_country /etc/nginx/GeoIP.dat; + geoip_city /etc/nginx/GeoLiteCity.dat; + geoip_proxy_recursive on; + + {{- if $cfg.enableVtsStatus }} + vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.vtsStatusZoneSize }}; + vhost_traffic_status_filter_by_set_key $geoip_country_code country::*; + {{ end -}} + + # lua section to return proper error codes when custom pages are used + lua_package_path '.?.lua;./etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/lua-resty-http/lib/?.lua;'; + init_by_lua_block { + require("error_page") + } + + sendfile on; + aio threads; + tcp_nopush on; + tcp_nodelay on; + + log_subrequest on; + + reset_timedout_connection on; + + keepalive_timeout {{ $cfg.keepAlive }}s; + + types_hash_max_size 2048; + server_names_hash_max_size {{ $cfg.serverNameHashMaxSize }}; + server_names_hash_bucket_size {{ $cfg.serverNameHashBucketSize }}; + + include /etc/nginx/mime.types; + default_type text/html; + {{ if $cfg.useGzip -}} + gzip on; + gzip_comp_level 5; + gzip_http_version 1.1; + gzip_min_length 256; + gzip_types {{ $cfg.gzipTypes }}; + gzip_proxied any; + {{- end }} + + client_max_body_size "{{ $cfg.bodySize }}"; + + log_format upstreaminfo '{{ if $cfg.useProxyProtocol }}$proxy_protocol_addr{{ else }}$remote_addr{{ end }} - ' + '[$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" ' + '$request_length $request_time $upstream_addr $upstream_response_length $upstream_response_time $upstream_status'; + + {{/* map urls that should not appear in access.log */}} + {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} + map $request $loggable { + {{- range $reqUri := $cfg.skipAccessLogUrls }} + {{ $reqUri }} 0;{{ end }} + default 1; + } + + access_log /var/log/nginx/access.log upstreaminfo if=$loggable; + error_log /var/log/nginx/error.log {{ $cfg.errorLogLevel }}; + + {{ if not (empty .defResolver) }}# Custom dns resolver. + resolver {{ .defResolver }} valid=30s; + {{ end }} + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # trust http_x_forwarded_proto headers correctly indicate ssl offloading + map $http_x_forwarded_proto $pass_access_scheme { + default $http_x_forwarded_proto; + '' $scheme; + } + + # Map a response error watching the header Content-Type + map $http_accept $httpAccept { + default html; + application/json json; + application/xml xml; + text/plain text; + } + + map $httpAccept $httpReturnType { + default text/html; + json application/json; + xml application/xml; + text text/plain; + } + + server_name_in_redirect off; + # This must be "on" if we're running Nginx ingress at a nodeport. + port_in_redirect on; + + ssl_protocols {{ $cfg.sslProtocols }}; + + # turn on session caching to drastically improve performance + {{ if $cfg.sslSessionCache }} + ssl_session_cache builtin:1000 shared:SSL:{{ $cfg.sslSessionCacheSize }}; + ssl_session_timeout {{ $cfg.sslSessionTimeout }}; + {{ end }} + + # allow configuring ssl session tickets + ssl_session_tickets {{ if $cfg.sslSessionTickets }}on{{ else }}off{{ end }}; + + # slightly reduce the time-to-first-byte + ssl_buffer_size {{ $cfg.sslBufferSize }}; + + {{ if not (empty $cfg.sslCiphers) }} + # allow configuring custom ssl ciphers + ssl_ciphers '{{ $cfg.sslCiphers }}'; + ssl_prefer_server_ciphers on; + {{ end }} + + {{ if not (empty .sslDHParam) }} + # allow custom DH file http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_dhparam + ssl_dhparam {{ .sslDHParam }}; + {{ end }} + + {{- if not $cfg.enableDynamicTlsRecords }} + ssl_dyn_rec_size_lo 0; + {{ end }} + + {{- if .customErrors }} + # Custom error pages + proxy_intercept_errors on; + {{ end }} + + {{- range $errCode := $cfg.customHttpErrors }} + error_page {{ $errCode }} = @custom_{{ $errCode }};{{ end }} + + # In case of errors try the next upstream server before returning an error + proxy_next_upstream error timeout invalid_header http_502 http_503 http_504{{ if $cfg.retryNonIdempotent }} non_idempotent{{ end }}; + + {{range $name, $upstream := .upstreams}} + upstream {{$upstream.Name}} { + {{ if $cfg.enableStickySessions -}} + sticky hash=sha1 httponly; + {{ else -}} + least_conn; + {{- end }} + {{ range $server := $upstream.Backends }}server {{ $server.Address }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }}; + {{ end }} + } + {{ end }} + + {{/* build all the required rate limit zones. Each annotation requires a dedicated zone */}} + {{/* 1MB -> 16 thousand 64-byte states or about 8 thousand 128-byte states */}} + {{- range $zone := (buildRateLimitZones .servers) }} + {{ $zone }} + {{ end }} + + {{ range $server := .servers }} + server { + server_name {{ $server.Name }}; + listen 80{{ if $cfg.useProxyProtocol }} proxy_protocol{{ end }}; + {{ if $server.SSL }}listen 443 {{ if $cfg.useProxyProtocol }}proxy_protocol{{ end }} ssl {{ if $cfg.enableSpdy }}spdy{{ end }} {{ if $cfg.useHttp2 }}http2{{ end }}; + {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} + # PEM sha: {{ $server.SSLPemChecksum }} + ssl_certificate {{ $server.SSLCertificate }}; + ssl_certificate_key {{ $server.SSLCertificateKey }}; + {{- end }} + + {{ if (and $server.SSL $cfg.hsts) -}} + more_set_headers "Strict-Transport-Security: max-age={{ $cfg.hstsMaxAge }}{{ if $cfg.hstsIncludeSubdomains }}; includeSubDomains{{ end }}; preload"; + {{- end }} + + {{ if $cfg.enableVtsStatus }}vhost_traffic_status_filter_by_set_key $geoip_country_code country::$server_name;{{ end }} + + {{- range $location := $server.Locations }} + {{ $path := buildLocation $location }} + location {{ $path }} { + {{ if gt (len $location.Whitelist.CIDR) 0 }} + {{- range $ip := $location.Whitelist.CIDR }} + allow {{ $ip }};{{ end }} + deny all; + {{ end -}} + + {{ if (and $server.SSL $location.Redirect.SSLRedirect) -}} + # enforce ssl on server side + if ($scheme = http) { + return 301 https://$host$request_uri; + } + {{- end }} + {{/* if the location contains a rate limit annotation, create one */}} + {{ $limits := buildRateLimit $location }} + {{- range $limit := $limits }} + {{ $limit }}{{ end }} + + {{ if $location.Auth.Secured }} + {{ if eq $location.Auth.Type "basic" }} + auth_basic "{{ $location.Auth.Realm }}"; + auth_basic_user_file {{ $location.Auth.File }}; + {{ else }} + #TODO: add nginx-http-auth-digest module + auth_digest "{{ $location.Auth.Realm }}"; + auth_digest_user_file {{ $location.Auth.File }}; + {{ end }} + proxy_set_header Authorization ""; + {{- end }} + + proxy_set_header Host $http_host; + + # Pass Real IP + proxy_set_header X-Real-IP $remote_addr; + + # Allow websocket connections + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-Proto $pass_access_scheme; + + # mitigate HTTPoxy Vulnerability + # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ + proxy_set_header Proxy ""; + + proxy_connect_timeout {{ $cfg.proxyConnectTimeout }}s; + proxy_send_timeout {{ $cfg.proxySendTimeout }}s; + proxy_read_timeout {{ $cfg.proxyReadTimeout }}s; + + proxy_redirect off; + proxy_buffering off; + + proxy_http_version 1.1; + + {{/* rewrite only works if the content is not compressed */}} + {{ if $location.Redirect.AddBaseURL -}} + proxy_set_header Accept-Encoding ""; + {{- end }} + + {{- buildProxyPass $location }} + } + {{ end }} + + {{ if eq $server.Name "_" }} + # this is required to avoid error if nginx is being monitored + # with an external software (like sysdig) + location /nginx_status { + allow 127.0.0.1; + deny all; + + access_log off; + stub_status on; + } + {{ end }} + {{ template "CUSTOM_ERRORS" $cfg }} + } + {{ end }} + + # default server, used for NGINX healthcheck and access to nginx stats + server { + # Use the port 18080 (random value just to avoid known ports) as default port for nginx. + # Changing this value requires a change in: + # https://github.com/kubernetes/contrib/blob/master/ingress/controllers/nginx/nginx/command.go#L104 + listen 18080 default_server reuseport backlog={{ .backlogSize }}; + + location /healthz { + access_log off; + return 200; + } + + location /nginx_status { + {{ if $cfg.enableVtsStatus -}} + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + {{ else }} + access_log off; + stub_status on; + {{- end }} + } + + location / { + proxy_pass http://upstream-default-backend; + } + {{- template "CUSTOM_ERRORS" $cfg }} + } + + # default server for services without endpoints + server { + listen 8181; + + location / { + {{ if .customErrors }} + content_by_lua_block { + openURL(503) + } + {{ else }} + return 503; + {{ end }} + } + } + } + + stream { + # TCP services + {{ range $i, $tcpServer := .tcpUpstreams }} + upstream tcp-{{ $tcpServer.Upstream.Name }} { + {{ range $server := $tcpServer.Upstream.Backends }}server {{ $server.Address }}:{{ $server.Port }}; + {{ end }} + } + + server { + listen {{ $tcpServer.Path }}; + proxy_connect_timeout {{ $cfg.proxyConnectTimeout }}; + proxy_timeout {{ $cfg.proxyReadTimeout }}; + proxy_pass tcp-{{ $tcpServer.Upstream.Name }}; + } + {{ end }} + + # UDP services + {{ range $i, $udpServer := .udpUpstreams }} + upstream udp-{{ $udpServer.Upstream.Name }} { + {{ range $server := $udpServer.Upstream.Backends }}server {{ $server.Address }}:{{ $server.Port }}; + {{ end }} + } + + server { + listen {{ $udpServer.Path }} udp; + proxy_timeout 10s; + proxy_responses 1; + proxy_pass udp-{{ $udpServer.Upstream.Name }}; + } + {{ end }} + } + + {{/* definition of templates to avoid repetitions */}} + {{ define "CUSTOM_ERRORS" }} + {{ range $errCode := .customHttpErrors }} + location @custom_{{ $errCode }} { + internal; + content_by_lua_block { + openURL({{ $errCode }}) + } + } + {{ end }} + {{ end }} diff --git a/common/tectonic/resources/manifests/ingress/nodeport/deployment.yaml b/common/tectonic/resources/manifests/ingress/nodeport/deployment.yaml new file mode 100644 index 0000000000..3aa330abc5 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/nodeport/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: tectonic-ingress-controller + labels: + app: tectonic-lb + component: ingress-controller + type: nginx +spec: + replicas: 1 + template: + metadata: + labels: + app: tectonic-lb + component: ingress-controller + type: nginx + spec: + containers: + - name: nginx-ingress-lb + image: ${ingress_controller_image} + args: + - /nginx-ingress-controller + - --nginx-configmap=$(POD_NAMESPACE)/tectonic-custom-error + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --default-ssl-certificate=tectonic-system/tectonic-ingress-tls-secret + - --watch-namespace=tectonic-system + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + - name: health + containerPort: 10254 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + volumeMounts: + - name: nginx-ingress-conf-template + mountPath: /etc/nginx/template + readOnly: true + volumes: + - name: nginx-ingress-conf-template + configMap: + name: tectonic-ingress-nginx-config + items: + - key: nginx.tmpl + path: nginx.tmpl + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 60 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/ingress/nodeport/service.yaml b/common/tectonic/resources/manifests/ingress/nodeport/service.yaml new file mode 100644 index 0000000000..b367a4c110 --- /dev/null +++ b/common/tectonic/resources/manifests/ingress/nodeport/service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + # keep it under 24 chars + name: tectonic-lb + labels: + app: tectonic-lb + component: ingress-controller +spec: + type: NodePort + selector: + app: tectonic-lb + component: ingress-controller + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 443 + nodePort: 32000 + - name: http + protocol: TCP + port: 80 + targetPort: 80 + nodePort: 32001 + - name: health + protocol: TCP + port: 10254 + targetPort: 10254 + nodePort: 32002 \ No newline at end of file diff --git a/common/tectonic/resources/manifests/monitoring/node-exporter-ds.yaml b/common/tectonic/resources/manifests/monitoring/node-exporter-ds.yaml new file mode 100644 index 0000000000..54ffcb835c --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/node-exporter-ds.yaml @@ -0,0 +1,46 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: tectonic-system +spec: + template: + metadata: + name: node-exporter + labels: + app: node-exporter + spec: + hostNetwork: true + hostPID: true + containers: + - image: ${node_exporter_image} + args: + - "-collector.procfs=/host/proc" + - "-collector.sysfs=/host/sys" + name: node-exporter + ports: + - containerPort: 9100 + hostPort: 9100 + name: scrape + resources: + requests: + memory: 30Mi + cpu: 100m + limits: + memory: 50Mi + cpu: 200m + volumeMounts: + - name: proc + readOnly: true + mountPath: /host/proc + - name: sys + readOnly: true + mountPath: /host/sys + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + diff --git a/common/tectonic/resources/manifests/monitoring/node-exporter-svc.yaml b/common/tectonic/resources/manifests/monitoring/node-exporter-svc.yaml new file mode 100644 index 0000000000..472b2af9bc --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/node-exporter-svc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: node-exporter + namespace: tectonic-system + labels: + app: node-exporter + k8s-app: node-exporter +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: 9100 + protocol: TCP + selector: + app: node-exporter + diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role-binding.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role-binding.yaml new file mode 100644 index 0000000000..041f6f6634 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: prometheus-k8s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-k8s +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: tectonic-system diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role.yaml new file mode 100644 index 0000000000..75b509dc5e --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-cluster-role.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRole +metadata: + name: prometheus-k8s +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s-config.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-config.yaml new file mode 100644 index 0000000000..a02694a1c9 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-config.yaml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-k8s + namespace: tectonic-system +data: + prometheus.yaml: | + alerting: + alertmanagers: [] + global: + evaluation_interval: 30s + scrape_interval: 30s + rule_files: + - /etc/prometheus/rules/*.rules + scrape_configs: + - job_name: kubelet + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - source_labels: [__meta_kubernetes_role] + action: replace + target_label: kubernetes_role + - source_labels: [__address__] + regex: '(.*):10250' + replacement: '${1}:10255' + target_label: __address__ + - job_name: tectonic-system/k8s-apps-http/0 + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: .+ + source_labels: + - __meta_kubernetes_service_label_k8s_app + - action: keep + regex: kube-system|tectonic-system + source_labels: + - __meta_kubernetes_namespace + - action: keep + regex: http-metrics + source_labels: + - __meta_kubernetes_endpoint_port_name + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - regex: "true" + replacement: "" + source_labels: + - __meta_kubernetes_service_annotation_alpha_monitoring_coreos_com_non_namespaced + target_label: namespace + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + replacement: svc_$1 + - action: replace + replacement: "" + target_label: __meta_kubernetes_pod_label_pod_template_hash + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + replacement: pod_$1 + - replacement: ${1} + source_labels: + - __meta_kubernetes_service_name + target_label: job + - regex: (.+) + replacement: ${1} + source_labels: + - __meta_kubernetes_service_label_k8s_app + target_label: job + - replacement: http-metrics + target_label: endpoint + scrape_interval: 15s diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s-rules.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-rules.yaml new file mode 100644 index 0000000000..bb2427179b --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-rules.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-k8s-rules + namespace: tectonic-system +data: + recording.rules: |+ + pod_name:container_memory_usage_bytes:sum = sum by(pod_name) ( + container_memory_usage_bytes{container_name!="POD",pod_name!=""} + ) + pod_name:container_spec_cpu_shares:sum = sum by(pod_name) ( + container_spec_cpu_shares{container_name!="POD",pod_name!=""} + ) + pod_name:container_fs_usage_bytes:sum = sum by(pod_name) ( + container_fs_usage_bytes{container_name!="POD",pod_name!=""} + ) + namespace:container_memory_usage_bytes:sum = sum by(namespace) ( + container_memory_usage_bytes{container_name!=""} + ) + namespace:container_spec_cpu_shares:sum = sum by(namespace) ( + container_spec_cpu_shares{container_name!=""} + ) + instance:node_cpu:rate:sum = sum by(instance) ( + rate(node_cpu{mode!="idle",mode!="iowait",mode!~"guest.*"}[1m]) + ) + instance:node_filesystem_usage:sum = sum by(instance) ( + (node_filesystem_size{mountpoint="/"} - node_filesystem_free{mountpoint="/"}) + ) + instance:node_network_receive_bytes:rate:sum = sum by(instance) ( + rate(node_network_receive_bytes[1m]) + ) + instance:node_network_transmit_bytes:rate:sum = sum by(instance) ( + rate(node_network_transmit_bytes[1m]) + ) + diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s-service-account.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-service-account.yaml new file mode 100644 index 0000000000..795b43761c --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-k8s + namespace: tectonic-system diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-k8s.json b/common/tectonic/resources/manifests/monitoring/prometheus-k8s.json new file mode 100644 index 0000000000..983097b1d9 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-k8s.json @@ -0,0 +1,26 @@ +{ + "apiVersion": "monitoring.coreos.com/v1alpha1", + "kind": "Prometheus", + "metadata": { + "name": "k8s", + "namespace": "tectonic-system", + "labels": { + "prometheus": "k8s" + } + }, + "spec": { + "replicas": 1, + "version": "${prometheus_version}", + "serviceAccountName": "prometheus-k8s", + "resources": { + "limits": { + "cpu": "400m", + "memory": "2000Mi" + }, + "requests": { + "cpu": "200m", + "memory": "1500Mi" + } + } + } +} diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role-binding.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role-binding.yaml new file mode 100644 index 0000000000..1139136a38 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRoleBinding +metadata: + name: prometheus-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-operator +subjects: +- kind: ServiceAccount + name: prometheus-operator + namespace: tectonic-system diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role.yaml new file mode 100644 index 0000000000..39e1a44893 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-operator-cluster-role.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1alpha1 +kind: ClusterRole +metadata: + name: prometheus-operator +rules: +- apiGroups: + - extensions + resources: + - thirdpartyresources + verbs: + - create +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - servicemonitors + verbs: + - "*" +- apiGroups: + - apps + resources: + - statefulsets + verbs: ["*"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["*"] +- apiGroups: [""] + resources: + - pods + verbs: ["list", "delete"] +- apiGroups: [""] + resources: + - services + - endpoints + verbs: ["get", "create", "update"] +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-operator-service-account.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-operator-service-account.yaml new file mode 100644 index 0000000000..9adcdad4bf --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-operator-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-operator + namespace: tectonic-system diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-operator.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-operator.yaml new file mode 100644 index 0000000000..f866260c41 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-operator.yaml @@ -0,0 +1,28 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: prometheus-operator + namespace: tectonic-system + labels: + operator: prometheus +spec: + replicas: 1 + template: + metadata: + labels: + operator: prometheus + spec: + serviceAccountName: prometheus-operator + containers: + - name: prometheus-operator + image: ${prometheus_operator_image} + args: + - "--config-reloader-image=${config_reload_image}" + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 200m + memory: 300Mi + diff --git a/common/tectonic/resources/manifests/monitoring/prometheus-svc.yaml b/common/tectonic/resources/manifests/monitoring/prometheus-svc.yaml new file mode 100644 index 0000000000..1cf59f2f09 --- /dev/null +++ b/common/tectonic/resources/manifests/monitoring/prometheus-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: tectonic-system + labels: + name: prometheus +spec: + selector: + prometheus: k8s + type: ClusterIP + ports: + - name: prometheus + port: 9090 + targetPort: 9090 + protocol: TCP + diff --git a/common/tectonic/resources/manifests/namespace.yaml b/common/tectonic/resources/manifests/namespace.yaml new file mode 100644 index 0000000000..ded393c7ae --- /dev/null +++ b/common/tectonic/resources/manifests/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tectonic-system \ No newline at end of file diff --git a/common/tectonic/resources/manifests/rbac/binding-admin.yaml b/common/tectonic/resources/manifests/rbac/binding-admin.yaml new file mode 100644 index 0000000000..68848ff3bb --- /dev/null +++ b/common/tectonic/resources/manifests/rbac/binding-admin.yaml @@ -0,0 +1,14 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: admin-user +subjects: + - kind: User + name: ${admin_email} + - kind: ServiceAccount + namespace: tectonic-system + name: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/common/tectonic/resources/manifests/rbac/binding-discovery.yaml b/common/tectonic/resources/manifests/rbac/binding-discovery.yaml new file mode 100644 index 0000000000..42d11fbb0a --- /dev/null +++ b/common/tectonic/resources/manifests/rbac/binding-discovery.yaml @@ -0,0 +1,13 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: discovery +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:discovery +subjects: +- kind: Group + name: 'system:unauthenticated' +- kind: Group + name: 'system:authenticated' diff --git a/common/tectonic/resources/manifests/rbac/role-admin.yaml b/common/tectonic/resources/manifests/rbac/role-admin.yaml new file mode 100644 index 0000000000..c937525e3d --- /dev/null +++ b/common/tectonic/resources/manifests/rbac/role-admin.yaml @@ -0,0 +1,10 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: admin +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] + - nonResourceURLs: ["*"] + verbs: ["*"] diff --git a/common/tectonic/resources/manifests/rbac/role-discovery.yaml b/common/tectonic/resources/manifests/rbac/role-discovery.yaml new file mode 100644 index 0000000000..96100bf305 --- /dev/null +++ b/common/tectonic/resources/manifests/rbac/role-discovery.yaml @@ -0,0 +1,9 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: discovery +rules: +- apiGroups: [] + resources: [] + nonResourceURLs: ['*'] + verbs: ['*'] diff --git a/common/tectonic/resources/manifests/rbac/role-user.yaml b/common/tectonic/resources/manifests/rbac/role-user.yaml new file mode 100644 index 0000000000..75b9b8bee3 --- /dev/null +++ b/common/tectonic/resources/manifests/rbac/role-user.yaml @@ -0,0 +1,67 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1alpha1 +metadata: + name: user +rules: + - apiGroups: [""] + resources: [ + "bindings", "configmaps", "events", "pods", "replicationcontrollers", + "secrets", "services", "serviceaccounts", + "pods/attach", + "pods/binding", + "pods/exec", + "pods/log", + "pods/portforward", + "pods/proxy", + "pods/status", + "replicationcontrollers/scale", + "replicationcontrollers/status", + "services/proxy", + "services/status" + ] + verbs: ["*"] + nonResourceURLs: [] + + - apiGroups: [""] + resources: [ + "componentstatuses", "endpoints", "limitranges", "nodes", "nodes/proxy", "nodes/status", + "namespaces", "namespaces/status", "namespaces/finalize", + "persistentvolumeclaims", "persistentvolumeclaims/status", "persistentvolumes", "resourcequotas", + "resourcequotas/status" + ] + verbs: ["get", "list", "watch", "proxy", "redirect"] + nonResourceURLs: [] + + - apiGroups: ["apps", "batch", "autoscaling", "policy"] + resources: ["*"] + verbs: ["*"] + nonResourceURLs: [] + + - apiGroups: ["extensions"] + resources: [ + "daemonsets", "deployments", "horizontalpodautoscalers", "ingresses", + "jobs", "replicasets", "replicationcontrollers", + + "daemonsets/status", + "deployments/rollback", + "deployments/scale", + "deployments/status", + "horizontalpodautoscalers/status", + "ingresses/status", + "jobs/status", + "replicasets/scale", + "replicasets/status", + "replicationcontrollers/scale" + ] + verbs: ["*"] + nonResourceURLs: [] + + - apiGroups: ["extensions"] + resources: ["networkpolicies", "thirdpartyresources"] + verbs: ["get", "list", "watch", "proxy", "redirect"] + nonResourceURLs: [] + + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["*"] + verbs: ["get", "list", "watch", "proxy", "redirect"] + nonResourceURLs: [] diff --git a/common/tectonic/resources/manifests/secrets/ca-cert.yaml b/common/tectonic/resources/manifests/secrets/ca-cert.yaml new file mode 100644 index 0000000000..d9c885e373 --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/ca-cert.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tectonic-ca-cert-secret + namespace: tectonic-system +type: Opaque +data: + ca-cert: ${ca_cert} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/secrets/identity-grpc-client.yaml b/common/tectonic/resources/manifests/secrets/identity-grpc-client.yaml new file mode 100644 index 0000000000..d4bf1234a6 --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/identity-grpc-client.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tectonic-identity-grpc-client-secret + namespace: tectonic-system +type: Opaque +data: + tls-cert: ${identity_client_tls_cert} + tls-key: ${identity_client_tls_key} + ca-cert: ${ca_cert} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/secrets/identity-grpc-server.yaml b/common/tectonic/resources/manifests/secrets/identity-grpc-server.yaml new file mode 100644 index 0000000000..4d6541ab32 --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/identity-grpc-server.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tectonic-identity-grpc-server-secret + namespace: tectonic-system +type: Opaque +data: + tls-cert: ${identity_server_tls_cert} + tls-key: ${identity_server_tls_key} + ca-cert: ${ca_cert} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/secrets/ingress-tls.yaml b/common/tectonic/resources/manifests/secrets/ingress-tls.yaml new file mode 100644 index 0000000000..32aa889976 --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/ingress-tls.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tectonic-ingress-tls-secret + namespace: tectonic-system +type: Opaque +data: + tls.crt: ${ingress_tls_cert} + tls.key: ${ingress_tls_key} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/secrets/license.json b/common/tectonic/resources/manifests/secrets/license.json new file mode 100644 index 0000000000..a52c75816f --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/license.json @@ -0,0 +1,11 @@ +{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "namespace": "tectonic-system", + "name": "tectonic-license-secret" + }, + "data": { + "license": "${license}" + } +} diff --git a/common/tectonic/resources/manifests/secrets/pull.json b/common/tectonic/resources/manifests/secrets/pull.json new file mode 100644 index 0000000000..df9c7ddd44 --- /dev/null +++ b/common/tectonic/resources/manifests/secrets/pull.json @@ -0,0 +1,12 @@ +{ + "apiVersion": "v1", + "kind": "Secret", + "type": "kubernetes.io/dockerconfigjson", + "metadata": { + "namespace": "tectonic-system", + "name": "coreos-pull-secret" + }, + "data": { + ".dockerconfigjson": "${pull_secret}" + } +} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/stats-emitter.yaml b/common/tectonic/resources/manifests/stats-emitter.yaml new file mode 100644 index 0000000000..ca2b14e236 --- /dev/null +++ b/common/tectonic/resources/manifests/stats-emitter.yaml @@ -0,0 +1,135 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: tectonic-stats-emitter + namespace: tectonic-system + labels: + app: tectonic-stats-emitter + component: stats-emitter +spec: + replicas: 1 + template: + metadata: + name: tectonic-stats-emitter + labels: + app: tectonic-stats-emitter + component: stats-emitter + annotations: + # TODO(squat): add backoff to stats-emitter so we don't need init pod. + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "tectonic-stats-extender-init", + "image": "${stats_extender_image}", + "imagePullPolicy": "IfNotPresent", + "command": [ + "/extender", + "--period=0s", + "--license=/etc/tectonic/licenses/license", + "--output=/etc/tectonic/stats/extensions", + "--extension=installerPlatform:$(INSTALLER_PLATFORM)", + "--extension=tectonicUpdaterEnabled:$(TECTONIC_UPDATER_ENABLED)", + "--extension=certificatesStrategy:$(CERTIFICATES_STRATEGY)" + ], + "env": [ + { + "name": "INSTALLER_PLATFORM", + "valueFrom": { + "configMapKeyRef": { + "key": "installerPlatform", + "name": "tectonic-config" + } + } + }, + { + "name": "CERTIFICATES_STRATEGY", + "valueFrom": { + "configMapKeyRef": { + "key": "certificatesStrategy", + "name": "tectonic-config" + } + } + }, + { + "name": "TECTONIC_UPDATER_ENABLED", + "valueFrom": { + "configMapKeyRef": { + "key": "tectonicUpdaterEnabled", + "name": "tectonic-config" + } + } + } + ], + "volumeMounts": [ + { + "name": "tectonic-license-secret", + "mountPath": "/etc/tectonic/licenses", + "readOnly": true + }, + { + "name": "tectonic-stats", + "mountPath": "/etc/tectonic/stats" + } + ] + } +]' + spec: + containers: + - name: tectonic-stats-emitter + imagePullPolicy: IfNotPresent + image: ${stats_emitter_image} + command: + - /spartakus + - volunteer + - --cluster-id=$(CLUSTER_ID) + - --database=https://stats-collector.tectonic.com + - --extensions=/etc/tectonic/stats/extensions + env: + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: tectonic-config + key: clusterID + volumeMounts: + - mountPath: /etc/tectonic/stats + name: tectonic-stats + readOnly: true + - name: tectonic-stats-extender + imagePullPolicy: IfNotPresent + image: ${stats_extender_image} + command: + - /extender + - --license=/etc/tectonic/licenses/license + - --output=/etc/tectonic/stats/extensions + - --extension=installerPlatform:$(INSTALLER_PLATFORM) + - --extension=tectonicUpdaterEnabled:$(TECTONIC_UPDATER_ENABLED) + - --extension=certificatesStrategy:$(CERTIFICATES_STRATEGY) + env: + - name: INSTALLER_PLATFORM + valueFrom: + configMapKeyRef: + name: tectonic-config + key: installerPlatform + - name: CERTIFICATES_STRATEGY + valueFrom: + configMapKeyRef: + name: tectonic-config + key: certificatesStrategy + - name: TECTONIC_UPDATER_ENABLED + valueFrom: + configMapKeyRef: + name: tectonic-config + key: tectonicUpdaterEnabled + volumeMounts: + - mountPath: /etc/tectonic/licenses + name: tectonic-license-secret + readOnly: true + - mountPath: /etc/tectonic/stats + name: tectonic-stats + volumes: + - name: tectonic-license-secret + secret: + secretName: tectonic-license-secret + - name: tectonic-stats + emptyDir: {} + imagePullSecrets: + - name: coreos-pull-secret \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/app-version-kind.yaml b/common/tectonic/resources/manifests/updater/app-version-kind.yaml new file mode 100644 index 0000000000..33fd020a6e --- /dev/null +++ b/common/tectonic/resources/manifests/updater/app-version-kind.yaml @@ -0,0 +1,7 @@ +apiVersion: "extensions/v1beta1" +kind: "ThirdPartyResource" +metadata: + name: "app-version.coreos.com" +description: "An experimental specification for Tectonic components' versions" +versions: + - name: "v1" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/app-version-kubernetes.json b/common/tectonic/resources/manifests/updater/app-version-kubernetes.json new file mode 100644 index 0000000000..23765f1a65 --- /dev/null +++ b/common/tectonic/resources/manifests/updater/app-version-kubernetes.json @@ -0,0 +1,19 @@ +{ + "apiVersion": "coreos.com/v1", + "kind": "AppVersion", + "metadata": { + "name": "kubernetes", + "namespace": "tectonic-system", + "labels": { + "managed-by-channel-operator": "true" + } + }, + "spec": { + "desiredVersion": "${kubernetes_version}", + "paused": false + }, + "status": { + "currentVersion": "${kubernetes_version}", + "paused": false + } +} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/app-version-tectonic-cluster.json b/common/tectonic/resources/manifests/updater/app-version-tectonic-cluster.json new file mode 100644 index 0000000000..282047ecfc --- /dev/null +++ b/common/tectonic/resources/manifests/updater/app-version-tectonic-cluster.json @@ -0,0 +1,19 @@ +{ + "apiVersion": "coreos.com/v1", + "kind": "AppVersion", + "metadata": { + "name": "tectonic-cluster", + "namespace": "tectonic-system", + "labels": { + "managed-by-channel-operator": "true" + } + }, + "spec": { + "desiredVersion": "${tectonic_version}", + "paused": false + }, + "status": { + "currentVersion": "${tectonic_version}", + "paused": false + } +} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/kube-version-operator.yaml b/common/tectonic/resources/manifests/updater/kube-version-operator.yaml new file mode 100644 index 0000000000..674bf08533 --- /dev/null +++ b/common/tectonic/resources/manifests/updater/kube-version-operator.yaml @@ -0,0 +1,21 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kube-version-operator + namespace: tectonic-system + labels: + k8s-app: kube-version-operator + managed-by-channel-operator: "true" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: kube-version-operator + tectonic-app-version-name: kubernetes + spec: + containers: + - name: kube-version-operator + image: ${kube_version_operator_image} + imagePullSecrets: + - name: coreos-pull-secret \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/migration-status-kind.yaml b/common/tectonic/resources/manifests/updater/migration-status-kind.yaml new file mode 100644 index 0000000000..cecad4043c --- /dev/null +++ b/common/tectonic/resources/manifests/updater/migration-status-kind.yaml @@ -0,0 +1,7 @@ +apiVersion: "extensions/v1beta1" +kind: "ThirdPartyResource" +metadata: + name: "migration-status.coreos.com" +description: "Resource to track migrations that have ran for a particular version." +versions: +- name: "v1" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/node-agent.yaml b/common/tectonic/resources/manifests/updater/node-agent.yaml new file mode 100644 index 0000000000..2ed6f80ed8 --- /dev/null +++ b/common/tectonic/resources/manifests/updater/node-agent.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: node-agent + namespace: tectonic-system + labels: + k8s-app: node-agent +spec: + template: + metadata: + labels: + k8s-app: node-agent + spec: + containers: + - name: node-agent + image: ${node_agent_image} + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: systemd + mountPath: /etc/systemd/system + - name: dbus + mountPath: /var/run/dbus + - name: etc-kubernetes + mountPath: /etc/kubernetes + imagePullSecrets: + - name: coreos-pull-secret + volumes: + - name: etc-kubernetes + hostPath: + path: /etc/kubernetes + - name: systemd + hostPath: + path: /etc/systemd/system + - name: dbus + hostPath: + path: /var/run/dbus \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/tectonic-channel-operator-config.json b/common/tectonic/resources/manifests/updater/tectonic-channel-operator-config.json new file mode 100644 index 0000000000..61a89ed4e5 --- /dev/null +++ b/common/tectonic/resources/manifests/updater/tectonic-channel-operator-config.json @@ -0,0 +1,15 @@ +{ + "apiVersion": "coreos.com/v1", + "kind": "ChannelOperatorConfig", + "metadata":{ + "name": "default", + "namespace": "tectonic-system" + }, + "server": "${update_server}", + "channel": "${update_channel}", + "appID": "${update_app_id}", + "automaticUpdate": false, + "triggerUpdate": false, + "triggerUpdateCheck": false, + "updateCheckInterval": 2700 +} \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/tectonic-channel-operator-kind.yaml b/common/tectonic/resources/manifests/updater/tectonic-channel-operator-kind.yaml new file mode 100644 index 0000000000..28a65f286a --- /dev/null +++ b/common/tectonic/resources/manifests/updater/tectonic-channel-operator-kind.yaml @@ -0,0 +1,7 @@ +apiVersion: "extensions/v1beta1" +kind: "ThirdPartyResource" +metadata: + name: "channel-operator-config.coreos.com" +description: "Tectonic Channel Operator Config" +versions: + - name: "v1" \ No newline at end of file diff --git a/common/tectonic/resources/manifests/updater/tectonic-channel-operator.yaml b/common/tectonic/resources/manifests/updater/tectonic-channel-operator.yaml new file mode 100644 index 0000000000..5e4bc3e951 --- /dev/null +++ b/common/tectonic/resources/manifests/updater/tectonic-channel-operator.yaml @@ -0,0 +1,34 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: tectonic-channel-operator + namespace: tectonic-system + labels: + k8s-app: tectonic-channel-operator + managed-by-channel-operator: "true" +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: tectonic-channel-operator + tectonic-app-version-name: tectonic-cluster + spec: + containers: + - name: tectonic-channel-operator + image: ${tectonic_channel_operator_image} + env: + - name: CLUSTER_ID + valueFrom: + configMapKeyRef: + name: tectonic-config + key: clusterID + volumeMounts: + - name: certs + mountPath: /etc/ssl/certs + imagePullSecrets: + - name: coreos-pull-secret + volumes: + - name: certs + hostPath: + path: /usr/share/ca-certificates \ No newline at end of file diff --git a/common/tectonic/resources/tectonic.sh b/common/tectonic/resources/tectonic.sh new file mode 100644 index 0000000000..e8765b3507 --- /dev/null +++ b/common/tectonic/resources/tectonic.sh @@ -0,0 +1,135 @@ +#!/bin/bash +set -e + +if [ "$#" -ne "2" ]; then + echo "Usage: $0 kubeconfig assets_path" + exit 1 +fi + +KUBECONFIG=$1 +ASSETS_PATH=$2 + +# Setup API Authentication +K8S_API=$(grep "server" $KUBECONFIG | cut -f 2- -d ":" | tr -d " ") +K8S_API_CA=$(mktemp); grep "certificate-authority-data" $KUBECONFIG | cut -f 2- -d ":" | tr -d " " | base64 -d > $K8S_API_CA +K8S_API_CERT=$(mktemp); grep "client-certificate-data" $KUBECONFIG | cut -f 2- -d ":" | tr -d " " | base64 -d > $K8S_API_CERT +K8S_API_KEY=$(mktemp); grep "client-key-data" $KUBECONFIG | cut -f 2- -d ":" | tr -d " " | base64 -d > $K8S_API_KEY +CURL="curl -sNL --cacert $K8S_API_CA --cert $K8S_API_CERT --key $K8S_API_KEY" +trap "rm -f $K8S_API_CA $K8S_API_CERT $K8S_API_KEY" EXIT + +# Setup helper functions +function wait_for_tpr() { + echo "Waiting for third-party resource definitions..." + until $CURL -f "$K8S_API/$1" &> /dev/null; do + sleep 5 + done +} + +function create_resource() { + STATUS=$($CURL -o /dev/null --write-out '%{http_code}\n' -H "Content-Type: application/$1" -d"$(cat $ASSETS_PATH/$2)" "$K8S_API/$3") + if [ "$STATUS" != "200" ] && [ "$STATUS" != "201" ] && [ "$STATUS" != "409" ]; then + echo -e "Failed to create $2 (got $STATUS): " >&2 + $CURL -H "Content-Type: application/$1" -d"$(cat $ASSETS_PATH/$2)" "$K8S_API/$3" >&2 + exit 1 + fi +} + +function delete_resource() { + $CURL -H "Content-Type: application/$1" -XDELETE "$K8S_API/$2" &> /dev/null +} + +# Wait for Kubernetes to be in a proper state +echo "Waiting for Kubernetes API..." +until $CURL -f "$K8S_API/version" &> /dev/null; do + sleep 5 +done + +echo "Waiting for Kubernetes components..." +while $CURL "$K8S_API/api/v1/namespaces/kube-system/pods" 2>/dev/null | grep Pending > /dev/null; do + sleep 5 +done +sleep 10 + +# Creating resources +echo "Creating Tectonic Namespace" +create_resource yaml namespace.yaml api/v1/namespaces + +echo "Creating Initial Roles" +delete_resource yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterroles/admin +create_resource yaml rbac/role-admin.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterroles +create_resource yaml rbac/role-user.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterroles +create_resource yaml rbac/binding-admin.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings +create_resource yaml rbac/binding-discovery.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings + +echo "Creating Tectonic ConfigMaps" +create_resource yaml config.yaml api/v1/namespaces/tectonic-system/configmaps + +echo "Creating Tectonic Secrets" +create_resource json secrets/pull.json api/v1/namespaces/tectonic-system/secrets +create_resource json secrets/license.json api/v1/namespaces/tectonic-system/secrets +create_resource yaml secrets/ingress-tls.yaml api/v1/namespaces/tectonic-system/secrets +create_resource yaml secrets/ca-cert.yaml api/v1/namespaces/tectonic-system/secrets +create_resource yaml secrets/identity-grpc-client.yaml api/v1/namespaces/tectonic-system/secrets +create_resource yaml secrets/identity-grpc-server.yaml api/v1/namespaces/tectonic-system/secrets + +echo "Creating Tectonic Identity" +create_resource yaml identity/configmap.yaml api/v1/namespaces/tectonic-system/configmaps +create_resource yaml identity/services.yaml api/v1/namespaces/tectonic-system/services +create_resource yaml identity/deployment.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments + +echo "Creating Tectonic Console" +create_resource yaml console/service.yaml api/v1/namespaces/tectonic-system/services +create_resource yaml console/deployment.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments + +echo "Creating Tectonic Monitoring" +create_resource yaml monitoring/prometheus-operator-service-account.yaml api/v1/namespaces/tectonic-system/serviceaccounts +create_resource yaml monitoring/prometheus-operator-cluster-role.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterroles +create_resource yaml monitoring/prometheus-operator-cluster-role-binding.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings +create_resource yaml monitoring/prometheus-k8s-service-account.yaml api/v1/namespaces/tectonic-system/serviceaccounts +create_resource yaml monitoring/prometheus-k8s-cluster-role.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterroles +create_resource yaml monitoring/prometheus-k8s-cluster-role-binding.yaml apis/rbac.authorization.k8s.io/v1alpha1/clusterrolebindings +create_resource yaml monitoring/prometheus-k8s-config.yaml api/v1/namespaces/tectonic-system/configmaps +create_resource yaml monitoring/prometheus-k8s-rules.yaml api/v1/namespaces/tectonic-system/configmaps +create_resource yaml monitoring/prometheus-svc.yaml api/v1/namespaces/tectonic-system/services +create_resource yaml monitoring/node-exporter-svc.yaml api/v1/namespaces/tectonic-system/services +create_resource yaml monitoring/node-exporter-ds.yaml apis/extensions/v1beta1/namespaces/tectonic-system/daemonsets +create_resource yaml monitoring/prometheus-operator.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments +wait_for_tpr apis/monitoring.coreos.com/v1alpha1/prometheuses +create_resource json monitoring/prometheus-k8s.json apis/monitoring.coreos.com/v1alpha1/namespaces/tectonic-system/prometheuses + +echo "Creating Ingress" +create_resource yaml ingress/default-backend/configmap.yaml api/v1/namespaces/tectonic-system/configmaps +create_resource yaml ingress/default-backend/service.yaml api/v1/namespaces/tectonic-system/services +create_resource yaml ingress/default-backend/deployment.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments +create_resource yaml ingress/ingress.yaml apis/extensions/v1beta1/namespaces/tectonic-system/ingresses + +if [ "${ingress_kind}" = "HostPort" ]; then + create_resource yaml ingress/hostport.yaml apis/extensions/v1beta1/namespaces/tectonic-system/daemonsets +elif [ "${ingress_kind}" = "NodePort" ]; then + create_resource yaml ingress/nodeport/configmap.yaml api/v1/namespaces/tectonic-system/configmaps + create_resource yaml ingress/nodeport/service.yaml api/v1/namespaces/tectonic-system/services + create_resource yaml ingress/nodeport/deployment.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments +else + echo "Unrecognized Ingress Kind: ${ingress_kind}" +fi + +echo "Creating Heapster / Stats Emitter" +create_resource yaml heapster/service.yaml api/v1/namespaces/kube-system/services +create_resource yaml heapster/deployment.yaml apis/extensions/v1beta1/namespaces/kube-system/deployments +create_resource yaml stats-emitter.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments + +echo "Creating Tectonic Updater" +create_resource yaml updater/tectonic-channel-operator-kind.yaml apis/extensions/v1beta1/thirdpartyresources +create_resource yaml updater/app-version-kind.yaml apis/extensions/v1beta1/thirdpartyresources +create_resource yaml updater/migration-status-kind.yaml apis/extensions/v1beta1/thirdpartyresources +create_resource yaml updater/node-agent.yaml apis/extensions/v1beta1/namespaces/tectonic-system/daemonsets +create_resource yaml updater/kube-version-operator.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments +create_resource yaml updater/tectonic-channel-operator.yaml apis/extensions/v1beta1/namespaces/tectonic-system/deployments +wait_for_tpr apis/coreos.com/v1/channeloperatorconfigs +create_resource json updater/tectonic-channel-operator-config.json apis/coreos.com/v1/namespaces/tectonic-system/channeloperatorconfigs +wait_for_tpr apis/coreos.com/v1/appversions +create_resource json updater/app-version-tectonic-cluster.json apis/coreos.com/v1/namespaces/tectonic-system/appversions +create_resource json updater/app-version-kubernetes.json apis/coreos.com/v1/namespaces/tectonic-system/appversions + +echo "Tectonic installation is done" +exit 0 \ No newline at end of file diff --git a/common/tectonic/variables.tf b/common/tectonic/variables.tf new file mode 100644 index 0000000000..1d4cc4c22b --- /dev/null +++ b/common/tectonic/variables.tf @@ -0,0 +1,94 @@ +variable "container_images" { + description = "Container images to use" + type = "map" +} + +variable "versions" { + description = "Versions of the components to use" + type = "map" +} + +variable "platform" { + description = "Platform on which Tectonic is being installed (e.g. bare-metal, aws)" + type = "string" +} + +variable "ingress_kind" { + description = "Type of Ingress mapping to use (e.g. HostPort, NodePort)" + type = "string" +} + +variable "license" { + description = "License issued to run Tectonic" + type = "string" +} + +variable "pull_secret" { + description = "Authentication secret to pull container images" + type = "string" +} + +variable "ca_generated" { + description = "Define whether the CA has been generated or user-provided" + type = "string" +} + +variable "ca_cert" { + description = "PEM-encoded CA certificate, used to generate Tectonic Console's server certificate" + type = "string" +} + +variable "ca_key_alg" { + description = "Algorithm used to generate ca_key" + type = "string" +} + +variable "ca_key" { + description = "PEM-encoded CA key, used to generate Tectonic Console's server certificate" + type = "string" +} + +variable "domain" { + description = "Base address used to access the Tectonic Console, without protocol nor trailing forward slash" + type = "string" +} + +variable "admin_email" { + description = "E-mail address used to login to the Tectonic Console" + type = "string" +} + +variable "admin_password_hash" { + description = "Password used to login to the Tectonic Console, hashed by bcrypt" + type = "string" +} + +variable "update_server" { + description = "Server contacted to pull updates from" + type = "string" +} + +variable "update_channel" { + description = "Channel to pull updates from" + type = "string" +} + +variable "update_app_id" { + description = "Application identifier to pull updates for" + type = "string" +} + +variable "console_client_id" { + description = "OIDC identifier for the Tectonic Console" + type = "string" +} + +variable "kubectl_client_id" { + description = "OIDC identifier for kubectl" + type = "string" +} + +variable "kube_apiserver_url" { + description = "URL used to reach kube-apiserver" + type = "string" +} \ No newline at end of file diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf index 6d216ae619..8064a85f67 100644 --- a/modules/aws/master-asg/ignition.tf +++ b/modules/aws/master-asg/ignition.tf @@ -19,25 +19,6 @@ resource "ignition_config" "master" { ] } -resource "ignition_config" "worker" { - files = [ - "${ignition_file.etcd-endpoints.id}", - "${ignition_file.kubeconfig.id}", - "${ignition_file.kubelet-env.id}", - "${ignition_file.ca-cert.id}", - "${ignition_file.client-cert.id}", - "${ignition_file.client-key.id}", - ] - - systemd = [ - "${ignition_systemd_unit.etcd-member.id}", - "${ignition_systemd_unit.docker.id}", - "${ignition_systemd_unit.locksmithd.id}", - "${ignition_systemd_unit.kubelet-worker.id}", - "${ignition_systemd_unit.wait-for-dns.id}", - ] -} - resource "ignition_systemd_unit" "docker" { name = "docker.service" enable = true @@ -60,12 +41,6 @@ resource "ignition_systemd_unit" "kubelet-master" { content = "${file("${path.module}/resources/master-kubelet.service")}" } -resource "ignition_systemd_unit" "kubelet-worker" { - name = "kubelet.service" - enable = true - content = "${file("${path.module}/resources/master-kubelet.service")}" -} - resource "ignition_systemd_unit" "bootkube" { name = "bootkube.service" enable = true diff --git a/modules/aws/master-asg/resources/master-kubelet.service b/modules/aws/master-asg/resources/master-kubelet.service index 1c710ebd1e..4cd096a18c 100644 --- a/modules/aws/master-asg/resources/master-kubelet.service +++ b/modules/aws/master-asg/resources/master-kubelet.service @@ -15,6 +15,7 @@ ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /var/lib/cni +ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ --kubeconfig=/etc/kubernetes/kubeconfig \ @@ -27,8 +28,10 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --allow-privileged \ --node-labels=master=true \ --minimum-container-ttl-duration=6m0s \ - --cluster_dns=10.3.0.10 \ + --cluster_dns=${cluster_dns} \ --cluster_domain=cluster.local \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --anonymous-auth=false \ --cloud-provider=aws ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid Restart=always diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf index e8f0fd0fe8..a323341b7b 100644 --- a/modules/aws/worker-asg/ignition.tf +++ b/modules/aws/worker-asg/ignition.tf @@ -1,29 +1,9 @@ -resource "ignition_config" "master" { - files = [ - "${ignition_file.etcd-endpoints.id}", - "${ignition_file.kubeconfig.id}", - "${ignition_file.kubelet-env.id}", - "${ignition_file.opt-bootkube.id}", - "${ignition_file.ca-cert.id}", - "${ignition_file.client-cert.id}", - "${ignition_file.client-key.id}", - ] - - systemd = [ - "${ignition_systemd_unit.etcd-member.id}", - "${ignition_systemd_unit.docker.id}", - "${ignition_systemd_unit.locksmithd.id}", - "${ignition_systemd_unit.kubelet-master.id}", - "${ignition_systemd_unit.wait-for-dns.id}", - "${ignition_systemd_unit.bootkube.id}", - ] -} - resource "ignition_config" "worker" { files = [ - "${ignition_file.etcd-endpoints.id}", "${ignition_file.kubeconfig.id}", "${ignition_file.kubelet-env.id}", + "${ignition_file.max-user-watches.id}", + "${ignition_file.etcd-endpoints.id}", "${ignition_file.ca-cert.id}", "${ignition_file.client-cert.id}", "${ignition_file.client-key.id}", @@ -54,16 +34,27 @@ resource "ignition_systemd_unit" "locksmithd" { ] } -resource "ignition_systemd_unit" "kubelet-master" { - name = "kubelet.service" - enable = true - content = "${file("${path.module}/resources/worker-kubelet.service")}" +data "template_file" "kubelet-worker" { + template = "${file("${path.module}/resources/worker-kubelet.service")}" + + vars { + cluster_dns = "${var.tectonic_kube_dns_service_ip}" + } } resource "ignition_systemd_unit" "kubelet-worker" { name = "kubelet.service" enable = true - content = "${file("${path.module}/resources/worker-kubelet.service")}" + content = "${data.template_file.kubelet-worker.rendered}" +} + +data "template_file" "etcd-member" { + template = "${file("${path.module}/resources/etcd-member.service")}" + + vars { + version = "${var.tectonic_versions["etcd"]}" + endpoints = "${join(",",module.etcd.endpoints)}" + } } resource "ignition_systemd_unit" "bootkube" { @@ -104,55 +95,19 @@ resource "ignition_systemd_unit" "etcd-member" { dropin = [ { - name = "40-etcd-gateway.conf" - - content = < /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ --kubeconfig=/etc/kubernetes/kubeconfig \ @@ -26,8 +27,10 @@ ExecStart=/usr/lib/coreos/kubelet-wrapper \ --pod-manifest-path=/etc/kubernetes/manifests \ --allow-privileged \ --minimum-container-ttl-duration=6m0s \ - --cluster_dns=10.3.0.10 \ + --cluster_dns=${cluster_dns} \ --cluster_domain=cluster.local \ + --client-ca-file=/etc/kubernetes/ca.crt \ + --anonymous-auth=false \ --cloud-provider=aws ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid Restart=always diff --git a/platform-aws-asg/resources/etcd-member.service b/platform-aws-asg/resources/etcd-member.service new file mode 100644 index 0000000000..165865bb14 --- /dev/null +++ b/platform-aws-asg/resources/etcd-member.service @@ -0,0 +1,6 @@ +[Service] +Environment="ETCD_IMAGE_TAG=${version}" +ExecStart= +ExecStart=/usr/lib/coreos/etcd-wrapper gateway start \ + --listen-addr=127.0.0.1:2379 \ + --endpoints=${endpoints} \ No newline at end of file diff --git a/platform-aws-asg/tectonic.tf b/platform-aws-asg/tectonic.tf new file mode 100644 index 0000000000..7e2e8b34fc --- /dev/null +++ b/platform-aws-asg/tectonic.tf @@ -0,0 +1,84 @@ +module "bootkube" { + source = "../common/bootkube" + cloud_provider = "aws" + + kube_apiserver_url = "https://${aws_route53_record.api-external.name}:443" + oidc_issuer_url = "https://${aws_route53_record.ingress-public.name}/identity" + + # Platform-independent variables wiring, do not modify. + container_images = "${var.tectonic_container_images}" + + ca_cert = "${var.tectonic_ca_cert}" + ca_key = "${var.tectonic_ca_key}" + ca_key_alg = "${var.tectonic_ca_key_alg}" + + service_cidr = "${var.tectonic_service_cidr}" + cluster_cidr = "${var.tectonic_cluster_cidr}" + + kube_apiserver_service_ip = "${var.tectonic_kube_apiserver_service_ip}" + kube_dns_service_ip = "${var.tectonic_kube_dns_service_ip}" + + advertise_address = "0.0.0.0" + anonymous_auth = "false" + + oidc_username_claim = "email" + oidc_groups_claim = "groups" + oidc_client_id = "tectonic-kubectl" + + etcd_servers = ["http://127.0.0.1:2379"] +} + +module "tectonic" { + source = "../common/tectonic" + platform = "aws" + + domain = "${aws_route53_record.ingress-public.name}" + kube_apiserver_url = "https://${aws_route53_record.api-external.name}:443" + + # Platform-independent variables wiring, do not modify. + container_images = "${var.tectonic_container_images}" + versions = "${var.tectonic_versions}" + + license = "${var.tectonic_license}" + pull_secret = "${var.tectonic_pull_secret}" + + admin_email = "${var.tectonic_admin_email}" + admin_password_hash = "${var.tectonic_admin_password_hash}" + + update_channel = "${var.tectonic_update_channel}" + update_app_id = "${var.tectonic_update_app_id}" + update_server = "${var.tectonic_update_server}" + + ca_generated = "${module.bootkube.ca_cert == "" ? false : true}" + ca_cert = "${module.bootkube.ca_cert}" + ca_key_alg = "${module.bootkube.ca_key_alg}" + ca_key = "${module.bootkube.ca_key}" + + console_client_id = "tectonic-console" + kubectl_client_id = "tectonic-kubectl" + ingress_kind = "NodePort" +} + +resource "null_resource" "tectonic" { + depends_on = ["module.tectonic", "aws_autoscaling_group.masters"] + + connection { + host = "${aws_elb.api-external.dns_name}" + user = "core" + agent = true + } + + provisioner "file" { + source = "${path.cwd}/generated" + destination = "$HOME/tectonic" + } + + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /opt", + "sudo rm -rf /opt/tectonic", + "sudo mv /home/core/tectonic /opt/", + "sudo systemctl start tectonic", + ] + } +} \ No newline at end of file