1
0
mirror of https://github.com/openshift/openshift-docs.git synced 2026-02-07 00:48:01 +01:00

GCP template fixes

This commit is contained in:
Cody Hoag
2020-10-14 21:08:07 -04:00
parent 9218fe203f
commit d389b7cfda
16 changed files with 254 additions and 195 deletions

View File

@@ -59,11 +59,18 @@ include::modules/installation-user-infra-exporting-common-variables.adoc[levelof
include::modules/installation-creating-gcp-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-vpc.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-dns.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-dns.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-lb.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-ext-lb.adoc[leveloffset=+2]
include::modules/installation-deployment-manager-int-lb.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-security.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-security.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-private-dns.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-private-dns.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-firewall-rules-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-firewall-rules.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-iam-shared-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-iam-shared-vpc.adoc[leveloffset=+2]
include::modules/installation-gcp-user-infra-rhcos.adoc[leveloffset=+1]

View File

@@ -58,11 +58,18 @@ include::modules/installation-user-infra-exporting-common-variables.adoc[levelof
include::modules/installation-creating-gcp-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-vpc.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-dns.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-dns.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-lb.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-ext-lb.adoc[leveloffset=+2]
include::modules/installation-deployment-manager-int-lb.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-security.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-security.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-private-dns.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-private-dns.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-firewall-rules-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-firewall-rules.adoc[leveloffset=+2]
include::modules/installation-creating-gcp-iam-shared-vpc.adoc[leveloffset=+1]
include::modules/installation-deployment-manager-iam-shared-vpc.adoc[leveloffset=+2]
include::modules/installation-gcp-user-infra-rhcos.adoc[leveloffset=+1]

View File

@@ -122,7 +122,7 @@ $ oc adm certificate approve <csr_name> <1>
+
[source,terminal]
----
$ oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve
$ oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs --no-run-if-empty oc adm certificate approve
----
+
[NOTE]

View File

@@ -1,6 +1,7 @@
// Module included in the following assemblies:
//
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
// * installing/installing_gcp/installing-restricted-networks-gcp.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
@@ -36,49 +37,19 @@ have to contact Red Hat support with your installation logs.
section of this topic and save it as `04_bootstrap.py` on your computer. This
template describes the bootstrap machine that your cluster requires.
. Export the variables that the deployment template uses:
//You need these variables before you deploy the load balancers for the shared VPC case, so the export statements that are if'd out for shared-vpc are in the load balancer module.
.. Export the control plane subnet location:
+
ifndef::shared-vpc[]
[source,terminal]
----
$ export CONTROL_SUBNET=`gcloud compute networks subnets describe ${INFRA_ID}-master-subnet --region=${REGION} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
.. Export the location of the {op-system-first} image that the installation program requires:
. Export the location of the {op-system-first} image that the installation program requires:
+
[source,terminal]
----
$ export CLUSTER_IMAGE=`gcloud compute images describe ${INFRA_ID}-rhcos-image --format json | jq -r .selfLink`
----
ifndef::shared-vpc[]
.. Export the three zones that the cluster uses:
+
[source,terminal]
----
$ export ZONE_0=`gcloud compute regions describe ${REGION} --format=json | jq -r .zones[0] | cut -d "/" -f9`
----
+
[source,terminal]
----
$ export ZONE_1=`gcloud compute regions describe ${REGION} --format=json | jq -r .zones[1] | cut -d "/" -f9`
----
+
[source,terminal]
----
$ export ZONE_2=`gcloud compute regions describe ${REGION} --format=json | jq -r .zones[2] | cut -d "/" -f9`
----
endif::shared-vpc[]
. Create a bucket and upload the `bootstrap.ign` file:
+
[source,terminal]
----
$ gsutil mb gs://${INFRA_ID}-bootstrap-ignition
$ gsutil cp bootstrap.ign gs://${INFRA_ID}-bootstrap-ignition/
$ gsutil cp <installation_directory>/bootstrap.ign gs://${INFRA_ID}-bootstrap-ignition/
----
. Create a signed URL for the bootstrap instance to use to access the Ignition
@@ -86,8 +57,7 @@ config. Export the URL from the output as a variable:
+
[source,terminal]
----
$ export BOOTSTRAP_IGN=`gsutil signurl -d 1h service-account-key.json \
gs://${INFRA_ID}-bootstrap-ignition/bootstrap.ign | grep "^gs:" | awk '{print $5}'`
$ export BOOTSTRAP_IGN=`gsutil signurl -d 1h service-account-key.json gs://${INFRA_ID}-bootstrap-ignition/bootstrap.ign | grep "^gs:" | awk '{print $5}'`
----
. Create a `04_bootstrap.yaml` resource definition file:
@@ -133,14 +103,22 @@ $ gcloud deployment-manager deployments create ${INFRA_ID}-bootstrap --config 04
ifndef::shared-vpc[]
. The templates do not manage load balancer membership due to limitations of Deployment
Manager, so you must add the bootstrap machine manually:
Manager, so you must add the bootstrap machine manually.
.. Add the bootstrap instance to the internal load balancer instance group:
+
[source,terminal]
----
$ gcloud compute target-pools add-instances \
${INFRA_ID}-api-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-bootstrap
$ gcloud compute target-pools add-instances \
${INFRA_ID}-ign-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-bootstrap
$ gcloud compute instance-groups unmanaged add-instances \
${INFRA_ID}-bootstrap-instance-group --zone=${ZONE_0} --instances=${INFRA_ID}-bootstrap
----
.. Add the bootstrap instance group to the internal load balancer backend service:
+
[source,terminal]
----
$ gcloud compute backend-services add-backend \
${INFRA_ID}-api-internal-backend-service --region=${REGION} --instance-group=${INFRA_ID}-bootstrap-instance-group --instance-group-zone=${ZONE_0}
----
endif::shared-vpc[]

View File

@@ -38,12 +38,11 @@ might have to contact Red Hat support with your installation logs.
section of this topic and save it as `05_control_plane.py` on your computer.
This template describes the control plane machines that your cluster requires.
. Export the following variables needed by the resource definition:
. Export the following variable required by the resource definition:
+
[source,terminal]
----
$ export MASTER_SERVICE_ACCOUNT_EMAIL=`gcloud iam service-accounts list | grep "^${INFRA_ID}-master-node " | awk '{print $2}'`
$ export MASTER_IGNITION=`cat master.ign`
$ export MASTER_IGNITION=`cat <installation_directory>/master.ign`
----
. Create a `05_control_plane.yaml` resource definition file:
@@ -68,7 +67,7 @@ resources:
image: '${CLUSTER_IMAGE}' <4>
machine_type: 'n1-standard-4' <5>
root_volume_size: '128'
service_account_email: '${MASTER_SERVICE_ACCOUNT_EMAIL}' <6>
service_account_email: '${MASTER_SERVICE_ACCOUNT}' <6>
ignition: '${MASTER_IGNITION}' <7>
EOF
@@ -89,14 +88,21 @@ $ gcloud deployment-manager deployments create ${INFRA_ID}-control-plane --confi
----
. The templates do not manage DNS entries due to limitations of Deployment
Manager, so you must add the etcd entries manually:
Manager, so you must add the etcd entries manually.
.. Export the following control plane variables:
+
[source,terminal]
----
$ export MASTER0_IP=`gcloud compute instances describe ${INFRA_ID}-master-0 --zone ${ZONE_0} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER1_IP=`gcloud compute instances describe ${INFRA_ID}-master-1 --zone ${ZONE_1} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER2_IP=`gcloud compute instances describe ${INFRA_ID}-master-2 --zone ${ZONE_2} --format json | jq -r .networkInterfaces[0].networkIP`
----
.. Add the DNS entries for the control plane etcd entries:
+
ifndef::shared-vpc[]
[source,terminal]
----
$ export MASTER0_IP=`gcloud compute instances describe ${INFRA_ID}-m-0 --zone ${ZONE_0} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER1_IP=`gcloud compute instances describe ${INFRA_ID}-m-1 --zone ${ZONE_1} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER2_IP=`gcloud compute instances describe ${INFRA_ID}-m-2 --zone ${ZONE_2} --format json | jq -r .networkInterfaces[0].networkIP`
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction add ${MASTER0_IP} --name etcd-0.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${INFRA_ID}-private-zone
@@ -113,9 +119,6 @@ endif::shared-vpc[]
ifdef::shared-vpc[]
[source,terminal]
----
$ export MASTER0_IP=`gcloud compute instances describe ${INFRA_ID}-m-0 --zone ${ZONE_0} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER1_IP=`gcloud compute instances describe ${INFRA_ID}-m-1 --zone ${ZONE_1} --format json | jq -r .networkInterfaces[0].networkIP`
$ export MASTER2_IP=`gcloud compute instances describe ${INFRA_ID}-m-2 --zone ${ZONE_2} --format json | jq -r .networkInterfaces[0].networkIP`
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
$ gcloud dns record-sets transaction add ${MASTER0_IP} --name etcd-0.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
@@ -130,46 +133,25 @@ $ gcloud dns record-sets transaction execute --zone ${INFRA_ID}-private-zone --p
----
endif::shared-vpc[]
ifndef::shared-vpc[]
. The templates do not manage load balancer membership due to limitations of Deployment
Manager, so you must add the control plane machines manually:
+
[source,terminal]
----
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-m-0
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_1}" --instances=${INFRA_ID}-m-1
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_2}" --instances=${INFRA_ID}-m-2
$ gcloud compute target-pools add-instances ${INFRA_ID}-ign-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-m-0
$ gcloud compute target-pools add-instances ${INFRA_ID}-ign-target-pool --instances-zone="${ZONE_1}" --instances=${INFRA_ID}-m-1
$ gcloud compute target-pools add-instances ${INFRA_ID}-ign-target-pool --instances-zone="${ZONE_2}" --instances=${INFRA_ID}-m-2
----
endif::shared-vpc[]
ifdef::shared-vpc[]
. The templates do not manage load balancer membership due to limitations of Deployment
Manager, so you must add the control plane machines manually.
** For an internal cluster, use the following commands:
** Run the following commands to add the control plane machines to the appropriate instance groups:
+
[source,terminal]
----
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_0}-instance-group --zone=${ZONE_0} --instances=${INFRA_ID}-m-0
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_1}-instance-group --zone=${ZONE_1} --instances=${INFRA_ID}-m-1
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_2}-instance-group --zone=${ZONE_2} --instances=${INFRA_ID}-m-2
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_0}-instance-group --zone=${ZONE_0} --instances=${INFRA_ID}-master-0
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_1}-instance-group --zone=${ZONE_1} --instances=${INFRA_ID}-master-1
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_2}-instance-group --zone=${ZONE_2} --instances=${INFRA_ID}-master-2
----
** For an external cluster, use the following commands:
** For an external cluster, you must also run the following commands to add the control plane machines to the target pools:
+
[source,terminal]
----
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_0}-instance-group --zone=${ZONE_0} --instances=${INFRA_ID}-m-0
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_1}-instance-group --zone=${ZONE_1} --instances=${INFRA_ID}-m-1
$ gcloud compute instance-groups unmanaged add-instances ${INFRA_ID}-master-${ZONE_2}-instance-group --zone=${ZONE_2} --instances=${INFRA_ID}-m-2
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-m-0
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_1}" --instances=${INFRA_ID}-m-1
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_2}" --instances=${INFRA_ID}-m-2
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-master-0
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_1}" --instances=${INFRA_ID}-master-1
$ gcloud compute target-pools add-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_2}" --instances=${INFRA_ID}-master-2
----
endif::shared-vpc[]
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:

View File

@@ -1,7 +1,12 @@
// Module included in the following assemblies:
//
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:shared-vpc:
endif::[]
[id="installation-creating-gcp-firewall-rules-vpc_{context}"]
= Creating firewall rules in GCP
@@ -55,7 +60,19 @@ EOF
. Create the deployment by using the `gcloud` CLI:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ gcloud deployment-manager deployments create ${INFRA_ID}-firewall --config 03_firewall.yaml --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ gcloud deployment-manager deployments create ${INFRA_ID}-firewall --config 03_firewall.yaml
----
endif::shared-vpc[]
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:
endif::[]

View File

@@ -1,7 +1,12 @@
// Module included in the following assemblies:
//
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:shared-vpc:
endif::[]
[id="installation-creating-gcp-iam-shared-vpc_{context}"]
= Creating IAM roles in GCP
@@ -57,71 +62,86 @@ $ gcloud deployment-manager deployments create ${INFRA_ID}-iam --config 03_iam.y
+
[source,terminal]
----
$ export MASTER_SA=`gcloud iam service-accounts list | grep "^${INFRA_ID}-master-node " | awk '{print $2}'`
$ export MASTER_SERVICE_ACCOUNT=`gcloud iam service-accounts list --filter "email~^${INFRA_ID}-m@${PROJECT_NAME}." --format json | jq -r '.[0].email'`
----
. Export the variable for the master service account:
. Export the variable for the worker service account:
+
[source,terminal]
----
$ export WORKER_SA=`gcloud iam service-accounts list | grep "^${INFRA_ID}-worker-node " | awk '{print $2}'`
$ export WORKER_SERVICE_ACCOUNT=`gcloud iam service-accounts list --filter "email~^${INFRA_ID}-w@${PROJECT_NAME}." --format json | jq -r '.[0].email'`
----
ifndef::shared-vpc[]
. Export the variable for the subnet that hosts the compute machines:
+
[source,terminal]
----
$ export COMPUTE_SUBNET=`gcloud compute networks subnets describe ${INFRA_ID}-worker-subnet --region=${REGION} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
ifdef::shared-vpc[]
. Assign the permissions that the installation program requires to the service accounts for the subnets that host the control plane and compute subnets:
.. Grant the `networkViewer` role of the project that hosts your shared VPC to the master service account:
+
[source,terminal]
----
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} projects add-iam-policy-binding ${HOST_PROJECT} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.networkViewer"
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} projects add-iam-policy-binding ${HOST_PROJECT} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.networkViewer"
----
.. Grant the `networkUser` role to the master service account for the control plane subnet:
+
[source,terminal]
----
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_CONTROL_SUBNET}" --member "serviceAccount:${MASTER_SA}" --role "roles/compute.networkUser" --region ${REGION}
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_CONTROL_SUBNET}" --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.networkUser" --region ${REGION}
----
.. Grant the `networkUser` role to the worker service account for the control plane subnet:
+
[source,terminal]
----
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_CONTROL_SUBNET}" --member "serviceAccount:${WORKER_SA}" --role "roles/compute.networkUser" --region ${REGION}
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_CONTROL_SUBNET}" --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/compute.networkUser" --region ${REGION}
----
.. Grant the `networkUser` role to the master service account for the compute subnet:
+
[source,terminal]
----
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_COMPUTE_SUBNET}" --member "serviceAccount:${MASTER_SA}" --role "roles/compute.networkUser" --region ${REGION}
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_COMPUTE_SUBNET}" --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.networkUser" --region ${REGION}
----
.. Grant the `networkUser` role to the worker service account for the compute subnet:
+
[source,terminal]
----
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_COMPUTE_SUBNET}" --member "serviceAccount:${WORKER_SA}" --role "roles/compute.networkUser" --region ${REGION}
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} compute networks subnets add-iam-policy-binding "${HOST_PROJECT_COMPUTE_SUBNET}" --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/compute.networkUser" --region ${REGION}
----
endif::shared-vpc[]
. The templates do not create the policy bindings due to limitations of Deployment
Manager, so you must create them manually:
+
[source,terminal]
----
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.instanceAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.networkAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.securityAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/iam.serviceAccountUser"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.instanceAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.networkAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.securityAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/iam.serviceAccountUser"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SA}" --role "roles/compute.viewer"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SA}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/compute.viewer"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/storage.admin"
----
. Create a service account key and store it locally for later use:
+
[source,terminal]
----
$ gcloud iam service-accounts keys create service-account-key.json --iam-account=${MASTER_SA}
$ gcloud iam service-accounts keys create service-account-key.json --iam-account=${MASTER_SERVICE_ACCOUNT}
----
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:
endif::[]

View File

@@ -1,7 +1,12 @@
// Module included in the following assemblies:
//
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:shared-vpc:
endif::[]
[id="installation-creating-gcp-lb_{context}"]
= Creating load balancers in GCP
@@ -39,17 +44,33 @@ requires.
.. Export the cluster network location:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ export CLUSTER_NETWORK=`gcloud compute networks describe ${HOST_PROJECT_NETWORK} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ export CLUSTER_NETWORK=`gcloud compute networks describe ${INFRA_ID}-network --format json | jq -r .selfLink`
----
endif::shared-vpc[]
.. Export the control plane subnet location:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ export CONTROL_SUBNET=`gcloud compute networks subnets describe ${HOST_PROJECT_CONTROL_SUBNET} --region=${REGION} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ export CONTROL_SUBNET=`gcloud compute networks subnets describe ${INFRA_ID}-master-subnet --region=${REGION} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
.. Export the three zones that the cluster uses:
+
@@ -68,83 +89,60 @@ $ export ZONE_1=`gcloud compute regions describe ${REGION} --format=json | jq -r
$ export ZONE_2=`gcloud compute regions describe ${REGION} --format=json | jq -r .zones[2] | cut -d "/" -f9`
----
. Create a `02_lb.yaml` resource definition file:
** For an internal cluster, run the following command:
. Create a `02_infra.yaml` resource definition file:
+
[source,terminal]
----
$ cat <<EOF >02_lb.yaml
imports:
- path: 02_lb_int.py
resources:
- name: cluster-lb-int
type: 02_lb_int.py
properties:
cluster_network: '${CLUSTER_NETWORK}' <1>
control_subnet: '${CONTROL_SUBNET}' <2>
infra_id: '${INFRA_ID}' <3>
region: '${REGION}' <4>
zones:
- '${ZONE_0}'
- '${ZONE_1}'
- '${ZONE_2}'
EOF
----
<1> `cluster_network` is the `selfLink` URL to the cluster network.
<2> `control_subnet` is the `selfLink` URL to the control subnet.
<3> `infra_id` is the `INFRA_ID` infrastructure name from the extraction step.
<4> `region` is the region to deploy the cluster into, for example `us-central1`.
** For an external cluster, run the following command:
+
[source,terminal]
----
$ cat <<EOF >02_lb.yaml
$ cat <<EOF >02_infra.yaml
imports:
- path: 02_lb_ext.py
- path: 02_lb_int.py
- path: 02_lb_int.py <1>
resources:
- name: cluster-lb-ext
- name: cluster-lb-ext <1>
type: 02_lb_ext.py
properties:
infra_id: '${INFRA_ID}' <1>
region: '${REGION}' <2>
infra_id: '${INFRA_ID}' <2>
region: '${REGION}' <3>
- name: cluster-lb-int
type: 02_lb_int.py
properties:
cluster_network: '${CLUSTER_NETWORK}' <3>
cluster_network: '${CLUSTER_NETWORK}'
control_subnet: '${CONTROL_SUBNET}' <4>
infra_id: '${INFRA_ID}' <1>
region: '${REGION}' <2>
zones:
infra_id: '${INFRA_ID}'
region: '${REGION}'
zones: <5>
- '${ZONE_0}'
- '${ZONE_1}'
- '${ZONE_2}'
EOF
----
<1> `infra_id` is the `INFRA_ID` infrastructure name from the extraction step.
<2> `region` is the region to deploy the cluster into, for example `us-central1`.
<3> `cluster_network` is the `selfLink` URL to the cluster network.
<4> `control_subnet` is the `selfLink` URL to the control subnet.
<1> Required only when deploying an external cluster.
<2> `infra_id` is the `INFRA_ID` infrastructure name from the extraction step.
<3> `region` is the region to deploy the cluster into, for example `us-central1`.
<4> `control_subnet` is the URI to the control subnet.
<5> `zones` are the zones to deploy the control plane instances into, like `us-east1-b`, `us-east1-c`, and `us-east1-d`.
. Create the deployment by using the `gcloud` CLI:
+
[source,terminal]
----
$ gcloud deployment-manager deployments create ${INFRA_ID}-lb --config 02_lb.yaml
$ gcloud deployment-manager deployments create ${INFRA_ID}-infra --config 02_infra.yaml
----
. Export the cluster IP address:
+
[source,terminal]
----
$ export CLUSTER_IP=$(gcloud compute addresses describe ${INFRA_ID}-cluster-ip --region=${REGION} --format json | jq -r .address)
$ export CLUSTER_IP=`gcloud compute addresses describe ${INFRA_ID}-cluster-ip --region=${REGION} --format json | jq -r .address`
----
. For an external cluster, also export the cluster public IP address:
+
[source,terminal]
----
$ export CLUSTER_PUBLIC_IP=$(gcloud compute addresses describe ${INFRA_ID}-cluster-public-ip --region=${REGION} --format json | jq -r .address)
$ export CLUSTER_PUBLIC_IP=`gcloud compute addresses describe ${INFRA_ID}-cluster-public-ip --region=${REGION} --format json | jq -r .address`
----
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:
endif::[]

View File

@@ -1,7 +1,12 @@
// Module included in the following assemblies:
//
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:shared-vpc:
endif::[]
[id="installation-creating-gcp-private-dns_{context}"]
= Creating a private DNS zone in GCP
@@ -53,16 +58,25 @@ EOF
. Create the deployment by using the `gcloud` CLI:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ gcloud deployment-manager deployments create ${INFRA_ID}-dns --config 02_dns.yaml --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ gcloud deployment-manager deployments create ${INFRA_ID}-dns --config 02_dns.yaml
----
endif::shared-vpc[]
. The templates do not create DNS entries due to limitations of Deployment
Manager, so you must create them manually:
.. Add the internal DNS entries:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
@@ -71,9 +85,21 @@ $ gcloud dns record-sets transaction add ${CLUSTER_IP} --name api.${CLUSTER_NAME
$ gcloud dns record-sets transaction add ${CLUSTER_IP} --name api-int.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
$ gcloud dns record-sets transaction execute --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction add ${CLUSTER_IP} --name api.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction add ${CLUSTER_IP} --name api-int.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction execute --zone ${INFRA_ID}-private-zone
----
endif::shared-vpc[]
.. For an external cluster, also add the external DNS entries:
+
ifdef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
@@ -81,3 +107,17 @@ $ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} dns record-
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} dns record-sets transaction add ${CLUSTER_PUBLIC_IP} --name api.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${BASE_DOMAIN_ZONE_NAME}
$ gcloud --account=${HOST_PROJECT_ACCOUNT} --project=${HOST_PROJECT} dns record-sets transaction execute --zone ${BASE_DOMAIN_ZONE_NAME}
----
endif::shared-vpc[]
ifndef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${BASE_DOMAIN_ZONE_NAME}
$ gcloud dns record-sets transaction add ${CLUSTER_PUBLIC_IP} --name api.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 60 --type A --zone ${BASE_DOMAIN_ZONE_NAME}
$ gcloud dns record-sets transaction execute --zone ${BASE_DOMAIN_ZONE_NAME}
----
endif::shared-vpc[]
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:
endif::[]

View File

@@ -78,14 +78,14 @@ $ gcloud deployment-manager deployments create ${INFRA_ID}-security --config 03_
+
[source,terminal]
----
$ export MASTER_SA=${INFRA_ID}-m@${PROJECT_NAME}.iam.gserviceaccount.com
$ export MASTER_SERVICE_ACCOUNT=`gcloud iam service-accounts list --filter "email~^${INFRA_ID}-m@${PROJECT_NAME}." --format json | jq -r '.[0].email'`
----
. Export the variable for the master service account:
. Export the variable for the worker service account:
+
[source,terminal]
----
$ export WORKER_SA=${INFRA_ID}-w@${PROJECT_NAME}.iam.gserviceaccount.com
$ export WORKER_SERVICE_ACCOUNT=`gcloud iam service-accounts list --filter "email~^${INFRA_ID}-w@${PROJECT_NAME}." --format json | jq -r '.[0].email'`
----
. The templates do not create the policy bindings due to limitations of Deployment
@@ -93,19 +93,19 @@ Manager, so you must create them manually:
+
[source,terminal]
----
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.instanceAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.networkAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/compute.securityAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/iam.serviceAccountUser"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SA}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.instanceAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.networkAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/compute.securityAdmin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/iam.serviceAccountUser"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${MASTER_SERVICE_ACCOUNT}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SA}" --role "roles/compute.viewer"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SA}" --role "roles/storage.admin"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/compute.viewer"
$ gcloud projects add-iam-policy-binding ${PROJECT_NAME} --member "serviceAccount:${WORKER_SERVICE_ACCOUNT}" --role "roles/storage.admin"
----
. Create a service account key and store it locally for later use:
+
[source,terminal]
----
$ gcloud iam service-accounts keys create service-account-key.json --iam-account=${MASTER_SA}
$ gcloud iam service-accounts keys create service-account-key.json --iam-account=${MASTER_SERVICE_ACCOUNT}
----

View File

@@ -142,6 +142,8 @@ $ export HOST_PROJECT_CONTROL_SUBNET=<control_plane_subnet>
----
$ export HOST_PROJECT_COMPUTE_SUBNET=<compute_subnet>
----
. Set up the shared VPC. See link:https://cloud.google.com/vpc/docs/provisioning-shared-vpc#setting_up[Setting up Shared VPC] in the GCP documentation.
endif::shared-vpc[]
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]

View File

@@ -55,7 +55,7 @@ endif::shared-vpc[]
ifdef::shared-vpc[]
[source,terminal]
----
$ export COMPUTE_SUBNET=$(gcloud compute networks subnets describe ${HOST_PROJECT_COMPUTE_SUBNET} --region=${REGION} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT} --format json | jq -r .selfLink)`
$ export COMPUTE_SUBNET=`gcloud compute networks subnets describe ${HOST_PROJECT_COMPUTE_SUBNET} --region=${REGION} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT} --format json | jq -r .selfLink`
----
endif::shared-vpc[]
@@ -63,14 +63,14 @@ endif::shared-vpc[]
+
[source,terminal]
----
$ export WORKER_SERVICE_ACCOUNT_EMAIL=`gcloud iam service-accounts list | grep "^${INFRA_ID}-worker-node " | awk '{print $2}'`
$ export WORKER_SERVICE_ACCOUNT=`gcloud iam service-accounts list --filter "email~^${INFRA_ID}-w@${PROJECT_NAME}." --format json | jq -r '.[0].email'`
----
.. Export the location of the compute machine Ignition config file:
+
[source,terminal]
----
$ export WORKER_IGNITION=`cat worker.ign`
$ export WORKER_IGNITION=`cat <installation_directory>/worker.ign`
----
. Create a `06_worker.yaml` resource definition file:
@@ -82,22 +82,31 @@ imports:
- path: 06_worker.py
resources:
- name: 'w-a-0' <1>
- name: 'worker-0' <1>
type: 06_worker.py
properties:
infra_id: '${INFRA_ID}' <2>
zone: '${ZONE_0}' <3>
compute_subnet: '${COMPUTE_SUBNET}' <4>
image: '${CLUSTER_IMAGE}' <5>
machine_type: 'n1-standard-4' <6>
root_volume_size: '128'
service_account_email: '${WORKER_SERVICE_ACCOUNT_EMAIL}' <7>
service_account_email: '${WORKER_SERVICE_ACCOUNT}' <7>
ignition: '${WORKER_IGNITION}' <8>
- name: 'worker-1'
type: 06_worker.py
properties:
infra_id: '${INFRA_ID}' <2>
zone: '${ZONE_1}' <3>
compute_subnet: '${COMPUTE_SUBNET}' <4>
image: '${CLUSTER_IMAGE}' <5>
machine_type: 'n1-standard-4' <6>
root_volume_size: '128'
service_account_email: '${WORKER_SERVICE_ACCOUNT}' <7>
ignition: '${WORKER_IGNITION}' <8>
EOF
----
<1> `name` is the name of the worker machine, for example `w-a-0`.
<1> `name` is the name of the worker machine, for example `worker-0`.
<2> `infra_id` is the `INFRA_ID` infrastructure name from the extraction step.
<3> `zone` is the zone to deploy the worker machine into, for example `us-central1-a`.
<4> `compute_subnet` is the `selfLink` URL to the compute subnet.

View File

@@ -60,6 +60,16 @@ $ export ROUTER_IP=`oc -n openshift-ingress get service router-default --no-head
----
... Add the A record to the private zones:
+
ifndef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction add ${ROUTER_IP} --name \*.apps.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 300 --type A --zone ${INFRA_ID}-private-zone
$ gcloud dns record-sets transaction execute --zone ${INFRA_ID}-private-zone
----
endif::shared-vpc[]
ifdef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
@@ -67,8 +77,19 @@ $ gcloud dns record-sets transaction start --zone ${INFRA_ID}-private-zone --pro
$ gcloud dns record-sets transaction add ${ROUTER_IP} --name \*.apps.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 300 --type A --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
$ gcloud dns record-sets transaction execute --zone ${INFRA_ID}-private-zone --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
----
endif::shared-vpc[]
... For an external cluster, also add the A record to the public zones:
+
ifndef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
$ gcloud dns record-sets transaction start --zone ${BASE_DOMAIN_ZONE_NAME}
$ gcloud dns record-sets transaction add ${ROUTER_IP} --name \*.apps.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 300 --type A --zone ${BASE_DOMAIN_ZONE_NAME}
$ gcloud dns record-sets transaction execute --zone ${BASE_DOMAIN_ZONE_NAME}
----
endif::shared-vpc[]
ifdef::shared-vpc[]
[source,terminal]
----
$ if [ -f transaction.yaml ]; then rm transaction.yaml; fi
@@ -76,6 +97,7 @@ $ gcloud dns record-sets transaction start --zone ${BASE_DOMAIN_ZONE_NAME} --pro
$ gcloud dns record-sets transaction add ${ROUTER_IP} --name \*.apps.${CLUSTER_NAME}.${BASE_DOMAIN}. --ttl 300 --type A --zone ${BASE_DOMAIN_ZONE_NAME} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
$ gcloud dns record-sets transaction execute --zone ${BASE_DOMAIN_ZONE_NAME} --project ${HOST_PROJECT} --account ${HOST_PROJECT_ACCOUNT}
----
endif::shared-vpc[]
** To add explicit domains instead of using a wildcard,
create entries for each of the cluster's current routes:

View File

@@ -38,5 +38,3 @@ The service account for the project that hosts the shared VPC network requires t
* Security Admin
* Network Management Admin
====
. Set up the shared VPC. See link:https://cloud.google.com/vpc/docs/provisioning-shared-vpc#setting_up[Setting up Shared VPC] in the GCP documentation.

View File

@@ -3,10 +3,6 @@
// * installing/installing_gcp/installing-gcp-user-infra.adoc
// * installing/installing_gcp/installing-gcp-user-infra-vpc.adoc
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:shared-vpc:
endif::[]
[id="installation-gcp-user-infra-wait-for-bootstrap_{context}"]
= Wait for bootstrap completion and remove bootstrap resources in GCP
@@ -44,28 +40,11 @@ If the command exits without a `FATAL` warning, your production control plane
has initialized.
. Delete the bootstrap resources:
ifndef::shared-vpc[]
+
[source,terminal]
----
$ gcloud compute target-pools remove-instances ${INFRA_ID}-api-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-bootstrap
$ gcloud compute target-pools remove-instances ${INFRA_ID}-ign-target-pool --instances-zone="${ZONE_0}" --instances=${INFRA_ID}-bootstrap
$ gsutil rm gs://${INFRA_ID}-bootstrap-ignition/bootstrap.ign
$ gsutil rb gs://${INFRA_ID}-bootstrap-ignition
$ gcloud deployment-manager deployments delete ${INFRA_ID}-bootstrap
----
endif::shared-vpc[]
ifdef::shared-vpc[]
+
[source,terminal]
----
$ gcloud compute backend-services remove-backend ${INFRA_ID}-api-internal-backend-service --region=${REGION} --instance-group=${INFRA_ID}-bootstrap-instance-group --instance-group-zone=${ZONE_0}
$ gsutil rm gs://${INFRA_ID}-bootstrap-ignition/bootstrap.ign
$ gsutil rb gs://${INFRA_ID}-bootstrap-ignition
$ gcloud deployment-manager deployments delete -q ${INFRA_ID}-bootstrap
$ gcloud deployment-manager deployments delete ${INFRA_ID}-bootstrap
----
endif::shared-vpc[]
ifeval::["{context}" == "installing-gcp-user-infra-vpc"]
:!shared-vpc:
endif::[]

View File

@@ -56,7 +56,7 @@ $ gsutil mb gs://$BUCKET/
+
[source,terminal]
----
$ PROJECT_ID=$(gcloud config get-value project)
$ PROJECT_ID=`gcloud config get-value project`
----
. Create a `velero` IAM service account:
@@ -71,9 +71,9 @@ $ gcloud iam service-accounts create velero \
+
[source,terminal]
----
$ SERVICE_ACCOUNT_EMAIL=$(gcloud iam service-accounts list \
$ SERVICE_ACCOUNT_EMAIL=`gcloud iam service-accounts list \
--filter="displayName:Velero Storage" \
--format 'value(email)')
--format 'value(email)'`
----
. Create the `ROLE_PERMISSIONS` variable: