From df2a420f41cc43fdd7e490ae66cf6444eba944ec Mon Sep 17 00:00:00 2001
From: Alex Dellapenta <adellape@redhat.com>
Date: Fri, 14 May 2021 14:25:14 -0600
Subject: [PATCH] Control pod placement using a project

---
 ...-taints-tolerations-adding-machineset.adoc |  2 +-
 ...s-scheduler-taints-tolerations-adding.adoc |  2 +-
 ...-scheduler-taints-tolerations-binding.adoc |  4 +-
 ...scheduler-taints-tolerations-projects.adoc | 45 +++++++++++++++++++
 ...scheduler-taints-tolerations-removing.adoc |  2 +-
 modules/olm-pod-placement.adoc                | 22 +++++++++
 .../nodes-scheduler-node-selectors.adoc       |  6 ++-
 .../nodes-scheduler-taints-tolerations.adoc   |  9 +++-
 .../olm-adding-operators-to-cluster.adoc      |  8 ++++
 9 files changed, 90 insertions(+), 10 deletions(-)
 create mode 100644 modules/nodes-scheduler-taints-tolerations-projects.adoc
 create mode 100644 modules/olm-pod-placement.adoc

diff --git a/modules/nodes-scheduler-taints-tolerations-adding-machineset.adoc b/modules/nodes-scheduler-taints-tolerations-adding-machineset.adoc
index e6547493a3..a4c5accfcc 100644
--- a/modules/nodes-scheduler-taints-tolerations-adding-machineset.adoc
+++ b/modules/nodes-scheduler-taints-tolerations-adding-machineset.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assemblies:
 //
-// * nodes/nodes-scheduler-taints-tolerations.adoc
+// * nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
 // * post_installation_configuration/node-tasks.adoc
 
 [id="nodes-scheduler-taints-tolerations-adding-machineset_{context}"]
diff --git a/modules/nodes-scheduler-taints-tolerations-adding.adoc b/modules/nodes-scheduler-taints-tolerations-adding.adoc
index 64dad9c756..e5ea1bb72a 100644
--- a/modules/nodes-scheduler-taints-tolerations-adding.adoc
+++ b/modules/nodes-scheduler-taints-tolerations-adding.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assemblies:
 //
-// * nodes/nodes-scheduler-taints-tolerations.adoc
+// * nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
 // * post_installation_configuration/node-tasks.adoc
 
 [id="nodes-scheduler-taints-tolerations-adding_{context}"]
diff --git a/modules/nodes-scheduler-taints-tolerations-binding.adoc b/modules/nodes-scheduler-taints-tolerations-binding.adoc
index a9173d4c08..b891a1862f 100644
--- a/modules/nodes-scheduler-taints-tolerations-binding.adoc
+++ b/modules/nodes-scheduler-taints-tolerations-binding.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assemblies:
 //
-// * nodes/nodes-scheduler-taints-tolerations.adoc
+// * nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
 // * post_installation_configuration/node-tasks.adoc
 
 [id="nodes-scheduler-taints-tolerations-bindings_{context}"]
@@ -24,5 +24,3 @@ $ oc adm taint nodes node1 dedicated=groupName:NoSchedule
 ----
 
 . Add a toleration to the pods by writing a custom admission controller.
-
-
diff --git a/modules/nodes-scheduler-taints-tolerations-projects.adoc b/modules/nodes-scheduler-taints-tolerations-projects.adoc
new file mode 100644
index 0000000000..779cc7aee4
--- /dev/null
+++ b/modules/nodes-scheduler-taints-tolerations-projects.adoc
@@ -0,0 +1,45 @@
+// Module included in the following assemblies:
+//
+// * nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
+// * post_installation_configuration/node-tasks.adoc
+
+[id="nodes-scheduler-taints-tolerations-projects_{context}"]
+= Creating a project with a node selector and toleration
+
+You can create a project that uses a node selector and toleration, which are set as annotations, to control the placement of pods onto specific nodes. Any subsequent resources created in the project are then scheduled on nodes that have a taint matching the toleration.
+
+.Prerequisites
+
+* A label for node selection has been added to one or more nodes by using a machine set or editing the node directly.
+* A taint has been added to one or more nodes by using a machine set or editing the node directly.
+
+.Procedure
+
+. Create a `Project` resource definition, specifying a node selector and toleration in the `metadata.annotations` section:
++
+.Example `project.yaml` file
+[source,yaml]
+----
+kind: Project
+apiVersion: project.openshift.io/v1
+metadata:
+  name: <project_name> <1>
+  annotations:
+    openshift.io/node-selector: '<label>' <2>
+    scheduler.alpha.kubernetes.io/defaultTolerations: >-
+      [{"operator": "Exists", "effect": "NoSchedule", "key":
+      "<key_name>"} <3>
+      ]
+----
+<1> The project name.
+<2> The default node selector label.
+<3> The toleration parameters, as described in the *Taint and toleration components* table. This example uses the `NoSchedule` effect, which allows existing pods on the node to remain, and the `Exists` operator, which does not take a value.
+
+. Use the `oc apply` command to create the project:
++
+[source,terminal]
+----
+$ oc apply -f project.yaml
+----
+
+Any subsequent resources created in the `<project_name>` namespace should now be scheduled on the specified nodes.
diff --git a/modules/nodes-scheduler-taints-tolerations-removing.adoc b/modules/nodes-scheduler-taints-tolerations-removing.adoc
index 921d7e551a..eca4a34a71 100644
--- a/modules/nodes-scheduler-taints-tolerations-removing.adoc
+++ b/modules/nodes-scheduler-taints-tolerations-removing.adoc
@@ -1,6 +1,6 @@
 // Module included in the following assemblies:
 //
-// * nodes/nodes-scheduler-taints-tolerations.adoc
+// * nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
 // * post_installation_configuration/node-tasks.adoc
 
 [id="nodes-scheduler-taints-tolerations-removing_{context}"]
diff --git a/modules/olm-pod-placement.adoc b/modules/olm-pod-placement.adoc
new file mode 100644
index 0000000000..dc13a7f5c1
--- /dev/null
+++ b/modules/olm-pod-placement.adoc
@@ -0,0 +1,22 @@
+// Module included in the following assemblies:
+//
+// * operators/admin/olm-adding-operators-to-cluster.adoc
+
+[id="olm-pod-placement_{context}"]
+= Pod placement of Operator workloads
+
+By default, Operator Lifecycle Manager (OLM) places pods on arbitrary worker nodes when installing an Operator or deploying Operand workloads. As an administrator, you can use projects with a combination of node selectors, taints, and tolerations to control the placement of Operators and Operands to specific nodes.
+
+Controlling pod placement of Operator and Operand workloads has the following prerequisites:
+
+. Determine a node or set of nodes to target for the pods per your requirements. If available, note an existing label, such as `node-role.kubernetes.io/app`, that identifies the node or nodes. Otherwise, add a label, such as `myoperator`, by using a machine set or editing the node directly. You will use this label in a later step as the node selector on your project.
+. If you want to ensure that only pods with a certain label are allowed to run on the nodes, while steering unrelated workloads to other nodes, add a taint to the node or nodes by using a machine set or editing the node directly. Use an effect that ensures that new pods that do not match the taint cannot be scheduled on the nodes. For example, a `myoperator:NoSchedule` taint ensures that new pods that do not match the taint are not scheduled onto that node, but existing pods on the node are allowed to remain.
+. Create a project that is configured with a default node selector and, if you added a taint, a matching toleration.
+
+At this point, the project you created can be used to steer pods towards the specified nodes in the following scenarios:
+
+For Operator pods::
+Administrators can create a `Subscription` object in the project. As a result, the Operator pods are placed on the specified nodes.
+
+For Operand pods::
+Using an installed Operator, users can create an application in the project, which places the custom resource (CR) owned by the Operator in the project. As a result, the Operand pods are placed on the specified nodes, unless the Operator is deploying cluster-wide objects or resources in other namespaces, in which case this customized pod placement does not apply.
diff --git a/nodes/scheduling/nodes-scheduler-node-selectors.adoc b/nodes/scheduling/nodes-scheduler-node-selectors.adoc
index 74e4f9edd6..1ce60b168c 100644
--- a/nodes/scheduling/nodes-scheduler-node-selectors.adoc
+++ b/nodes/scheduling/nodes-scheduler-node-selectors.adoc
@@ -9,7 +9,7 @@ toc::[]
 
 A _node selector_ specifies a map of key/value pairs that are defined using custom labels on nodes and selectors specified in pods.
 
-For the pod to be eligible to run on a node, the pod must have the same key/value node selector as the label on the node. 
+For the pod to be eligible to run on a node, the pod must have the same key/value node selector as the label on the node.
 
 // The following include statements pull in the module files that comprise
 // the assembly. Include any combination of concept, procedure, or reference
@@ -24,6 +24,8 @@ include::modules/nodes-scheduler-node-selectors-pod.adoc[leveloffset=+1]
 include::modules/nodes-scheduler-node-selectors-cluster.adoc[leveloffset=+1]
 
 include::modules/nodes-scheduler-node-selectors-project.adoc[leveloffset=+1]
+.Additional resources
+
+* xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-projects_nodes-scheduler-taints-tolerations[Creating a project with a node selector and toleration]
 
 // include::modules/nodes-scheduler-node-selectors-configuring.adoc[leveloffset=+1]
-
diff --git a/nodes/scheduling/nodes-scheduler-taints-tolerations.adoc b/nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
index 7cba42cd34..9ab4295950 100644
--- a/nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
+++ b/nodes/scheduling/nodes-scheduler-taints-tolerations.adoc
@@ -22,10 +22,15 @@ include::modules/nodes-scheduler-taints-tolerations-adding-machineset.adoc[level
 
 include::modules/nodes-scheduler-taints-tolerations-binding.adoc[leveloffset=+2]
 
+include::modules/nodes-scheduler-taints-tolerations-projects.adoc[leveloffset=+2]
+.Additional resources
+
+* Adding taints and tolerations xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-adding_nodes-scheduler-taints-tolerations[manually to nodes] or xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-adding-machineset_nodes-scheduler-taints-tolerations[with machine sets]
+* xref:../../nodes/scheduling/nodes-scheduler-node-selectors.adoc#nodes-scheduler-node-selectors-project_nodes-scheduler-node-selectors[Creating project-wide node selectors]
+* xref:../../operators/admin/olm-adding-operators-to-cluster.adoc#olm-pod-placement_olm-adding-operators-to-a-cluster[Pod placement of Operator workloads]
+
 include::modules/nodes-scheduler-taints-tolerations-special.adoc[leveloffset=+2]
 
 include::modules/nodes-scheduler-taints-tolerations-removing.adoc[leveloffset=+1]
 
 //Removed per upstream docs modules/nodes-scheduler-taints-tolerations-evictions.adoc[leveloffset=+1]
-
-
diff --git a/operators/admin/olm-adding-operators-to-cluster.adoc b/operators/admin/olm-adding-operators-to-cluster.adoc
index c8fbeaebda..a2a29032fd 100644
--- a/operators/admin/olm-adding-operators-to-cluster.adoc
+++ b/operators/admin/olm-adding-operators-to-cluster.adoc
@@ -33,4 +33,12 @@ include::modules/olm-installing-specific-version-cli.adoc[leveloffset=+1]
 .Additional resources
 
 * xref:../../operators/admin/olm-upgrading-operators.adoc#olm-approving-pending-upgrade_olm-upgrading-operators[Manually approving a pending Operator upgrade]
+
+include::modules/olm-pod-placement.adoc[leveloffset=+1]
+
+.Additional resources
+
+* Adding taints and tolerations xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-adding_nodes-scheduler-taints-tolerations[manually to nodes] or xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-adding-machineset_nodes-scheduler-taints-tolerations[with machine sets]
+* xref:../../nodes/scheduling/nodes-scheduler-node-selectors.adoc#nodes-scheduler-node-selectors-project_nodes-scheduler-node-selectors[Creating project-wide node selectors]
+* xref:../../nodes/scheduling/nodes-scheduler-taints-tolerations.adoc#nodes-scheduler-taints-tolerations-projects_nodes-scheduler-taints-tolerations[Creating a project with a node selector and toleration]
 endif::[]