From f46753637eab174f2e75ff3d78285700d5f92a2b Mon Sep 17 00:00:00 2001 From: Jason Boxman Date: Tue, 29 Oct 2019 16:16:56 -0400 Subject: [PATCH] Add content for SR-IOV Network Operator --- modules/nw-multus-add-pod.adoc | 20 ++- modules/nw-sriov-configuring-device.adoc | 82 ++++++++++ modules/nw-sriov-device-discovery.adoc | 77 +++++++++ modules/nw-sriov-installing-operator.adoc | 148 ++++++++++++++++++ modules/nw-sriov-network-attachment.adoc | 57 +++++++ modules/nw-sriov-supported-devices.adoc | 13 ++ .../multiple-networks/configuring-sr-iov.adoc | 116 +++++++++++++- 7 files changed, 510 insertions(+), 3 deletions(-) create mode 100644 modules/nw-sriov-configuring-device.adoc create mode 100644 modules/nw-sriov-device-discovery.adoc create mode 100644 modules/nw-sriov-installing-operator.adoc create mode 100644 modules/nw-sriov-network-attachment.adoc create mode 100644 modules/nw-sriov-supported-devices.adoc diff --git a/modules/nw-multus-add-pod.adoc b/modules/nw-multus-add-pod.adoc index 7e6c5f7759..42e1372a63 100644 --- a/modules/nw-multus-add-pod.adoc +++ b/modules/nw-multus-add-pod.adoc @@ -2,17 +2,31 @@ // // * networking/multiple-networks/attaching-pod.adoc +ifeval::["{context}" == "configuring-sr-iov"] +:sriov: +endif::[] + [id="nw-multus-add-pod_{context}"] = Adding a Pod to an additional network You can add a Pod to an additional network. The Pod continues to send normal cluster related network traffic over the default network. +ifdef::sriov[] +[NOTE] +===== +The Network Resources Injector will inject the `resource` parameter into the Pod CR automatically if a NetworkAttachmentDefinition CR associated with the SR-IOV CNI plug-in is specified. +===== +endif::sriov[] + .Prerequisites * The Pod must be in the same namespace as the additional network. * Install the OpenShift Command-line Interface (CLI), commonly known as `oc`. * You must log in to the cluster. +ifdef::sriov[] +* You must have the SR-IOV Operator installed and a SriovNetwork CR defined. +endif::sriov[] .Procedure @@ -63,7 +77,7 @@ spec: following command. Replace `` with the name of the Pod. + ---- -$ oc describe pod +$ oc get pod -o yaml ---- + In the following example, the `example-pod` Pod is attached to the `net1` @@ -104,3 +118,7 @@ status: <1> The `k8s.v1.cni.cncf.io/networks-status` parameter is a JSON array of objects. Each object describes the status of an additional network attached to the Pod. The annotation value is stored as a plain text value. + +ifeval::["{context}" == "configuring-sr-iov"] +:!sriov: +endif::[] diff --git a/modules/nw-sriov-configuring-device.adoc b/modules/nw-sriov-configuring-device.adoc new file mode 100644 index 0000000000..1ee599478b --- /dev/null +++ b/modules/nw-sriov-configuring-device.adoc @@ -0,0 +1,82 @@ +// Module included in the following assemblies: +// +// * networking/multiple-networks/configuring-sr-iov.adoc + +[id="configuring-sr-iov-devices_{context}"] += Configuring SR-IOV network devices + +The SR-IOV Network Operator adds the `SriovNetworkNodePolicy.sriovnetwork.openshift.io` Custom Resource Definition (CRD) to {product-title}. +You can configure the SR-IOV network device by creating a SriovNetworkNodePolicy Custom Resource (CR). + +[NOTE] +===== +When applying the configuration specified in a SriovNetworkNodePolicy CR, the SR-IOV Operator may drain the nodes, and in some cases, reboot nodes. +It may take several minutes for a configuration change to apply. +Ensure that there are enough available nodes in your cluster to handle the evicted workload beforehand. + +After the configuration update is applied, all the Pods in `sriov-network-operator` namespace will change to a `Running` status. +===== + +.Prerequisites + +* Install the OpenShift Command-line Interface (CLI), commonly known as `oc`. +* Log in as a user with `cluster-admin` privileges. +* You must have installed the SR-IOV Operator. + +.Procedure + +. Create the following SriovNetworkNodePolicy CR, and then save the YAML in the `-sriov-node-network.yaml` file. Replace `` with the name for this configuration. ++ +[source,yaml] +---- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: <1> + namespace: sriov-network-operator <2> +spec: + resourceName: <3> + nodeSelector: + feature.node.kubernetes.io/network-sriov.capable: "true" <4> + priority: <5> + mtu: <6> + numVfs: <7> + nicSelector: <8> + vendor: "" <9> + deviceID: "" <10> + pfName: ["", ...] <11> + rootDevices: ["", "..."] <12> + deviceType: <13> + isRdma: false <14> +---- +<1> Specify a name for the CR. +<2> Specify the namespace where the SR-IOV Operator is installed. +<3> Specify the resource name of the SR-IOV device plug-in. The prefix `openshift.io/` will be added when it's referred in Pod spec. You can create multiple SriovNetworkNodePolicy CRs for a resource name. +<4> Specify the node selector to select which node to be configured. User can choose to label the nodes manually or with tools like Kubernetes Node Feature Discovery. +Only SR-IOV network devices on selected nodes will be configured. The SR-IOV +CNI plug-in and device plug-in will be only deployed on selected nodes. +<5> Specify an integer value between `0` and `99`. A larger number gets lower priority, so a priority of `99` is lower than a priority of `10`. +<6> Specify a value for the maximum transmission unit (MTU) of the virtual function. The value for MTU must be in the range from `1` to `9000`. If you do not need to specify the MTU, specify a value of `''`. +<7> Specify the number of the virtual functions (VF) to create for the SR-IOV physical network device. For an Intel Network Interface Card (NIC), the number of VFs cannot be larger than the total VFs supported by the device. For a Mellanox NIC, the number of VFs cannot be larger than `128`. +<8> The `nicSelector` mapping selects the Ethernet device for the Operator to configure. You do not need to specify values for all the parameters. It is recommended to identify the Ethernet adapter with enough precision to minimize the possibility of selecting an Ethernet device unintentionally. +If you specify `rootDevices`, you must also specify a value for `vendor`, `deviceID`, or `pfName`. +If you specify both `pfName` and `rootDevices` at the same time, ensure that they point to an identical device. +<9> Specify the vendor hex code of the SR-IOV network device. The only allowed values are either `8086` or `15b3`. +<10> Specify the device hex code of SR-IOV network device. The only allowed values are `158b`, `1015`, `1017`. +<11> The parameter accepts an array of one or more physical function (PF) names for the Ethernet device. +<12> The parameter accepts an array of one or more PCI bus addresses for the physical function of the Ethernet device. Provide the address in the following format: `0000:02:00.1`. +<13> Specify the driver type for the virtual functions. You can specify one of the following values: `netdevice` or `vfio-pci`. The default value is `netdevice`. +<14> Specify whether to enable RDMA mode. The default value is `false`. Only RDMA over Converged Ethernet (RoCE) mode is supported on Mellanox Ethernet adapters. ++ +[NOTE] +===== +If `RDMA` flag is set to `true`, you can continue to use the RDMA enabled VF as a normal network device. +A device can be used in either mode. +===== + +. Create the CR by running the following command: ++ +---- +$ oc create -f <1> +---- +<1> Replace `` with the name of the file you created in the previous step. diff --git a/modules/nw-sriov-device-discovery.adoc b/modules/nw-sriov-device-discovery.adoc new file mode 100644 index 0000000000..57ef77db23 --- /dev/null +++ b/modules/nw-sriov-device-discovery.adoc @@ -0,0 +1,77 @@ +// Module included in the following assemblies: +// +// * networking/multiple-networks/configuring-sr-iov.adoc + +[id="discover-sr-iov-devices_{context}"] += Automated discovery of SR-IOV network devices + +The SR-IOV Network Operator will search your cluster for SR-IOV capable network devices on worker nodes. +The Operator creates and updates a SriovNetworkNodeState Custom Resource (CR) for each worker node that provides a compatible SR-IOV network device. + +One CR is created for each worker node, and shares the same name as the node. +The `.spec.interfaces` list provides information about the network devices on a node. + +[IMPORTANT] +==== +Do not modify a SriovNetworkNodeState CR. +The Operator creates and manages these resources automatically. +==== + +The following is an example of a SriovNetworkNodeState CR created by the SR-IOV Network Operator: + +[source,yaml] +---- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodeState +metadata: + name: node-25 <1> + namespace: sriov-network-operator + ownerReferences: + - apiVersion: sriovnetwork.openshift.io/v1 + blockOwnerDeletion: true + controller: true + kind: SriovNetworkNodePolicy + name: default +spec: + dpConfigVersion: d41d8cd98f00b204e9800998ecf8427e +status: + interfaces: <2> + - deviceID: "1017" + driver: mlx5_core + mtu: 1500 + name: ens785f0 + pciAddress: "0000:18:00.0" + totalvfs: 8 + vendor: 15b3 + - deviceID: "1017" + driver: mlx5_core + mtu: 1500 + name: ens785f1 + pciAddress: "0000:18:00.1" + totalvfs: 8 + vendor: 15b3 + - deviceID: 158b + driver: i40e + mtu: 1500 + name: ens817f0 + pciAddress: 0000:81:00.0 + totalvfs: 64 + vendor: "8086" + - deviceID: 158b + driver: i40e + mtu: 1500 + name: ens817f1 + pciAddress: 0000:81:00.1 + totalvfs: 64 + vendor: "8086" + - deviceID: 158b + driver: i40e + mtu: 1500 + name: ens803f0 + pciAddress: 0000:86:00.0 + totalvfs: 64 + vendor: "8086" + syncStatus: Succeeded +---- +<1> The value for the `name` parameter is the same as the name of the worker node. +<2> The `interfaces` collection includes a list of all of the SR-IOV devices discovered by the Operator on the worker node. diff --git a/modules/nw-sriov-installing-operator.adoc b/modules/nw-sriov-installing-operator.adoc new file mode 100644 index 0000000000..94fd4fe57f --- /dev/null +++ b/modules/nw-sriov-installing-operator.adoc @@ -0,0 +1,148 @@ +// Module included in the following assemblies: +// +// * networking/multiple-networks/configuring-sr-iov.adoc + +[id="installing-sr-iov-operator_{context}"] += Installing SR-IOV Network Operator + +As a cluster administrator, you can install the SR-IOV Network Operator using the {product-title} CLI or the web console. + +[id="install-operator-cli"] +== Installing the Operator using the CLI + +As a cluster administrator, you can install the Operator using the CLI. + +.Prerequisites + +* A cluster installed on bare-metal hardware with nodes that have hardware that supports SR-IOV. +* The {product-title} Command-line Interface (CLI), commonly known as `oc`. +* Log in as a user with `cluster-admin` privileges. + +.Procedure + +. Create a namespace for the SR-IOV Network Operator by completing the following actions: + +.. Create the following Namespace Custom Resource (CR) that defines the `sriov-network-operator` namespace, and then save the YAML in the `sriov-namespace.yaml` file: ++ +---- +apiVersion: v1 +kind: Namespace +metadata: + name: sriov-network-operator + labels: + openshift.io/run-level: "1" +---- + +.. Create the namespace by running the following command: ++ +---- +$ oc create -f sriov-namespace.yaml +---- + +. Install the SR-IOV Network Operator in the namespace you created in the previous step by creating the following objects: + +.. Create the following OperatorGroup CR and save the YAML in the +`sriov-operatorgroup.yaml` file: ++ +---- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: sriov-network-operators + namespace: sriov-network-operator +spec: + targetNamespaces: + - sriov-network-operator +---- + +.. Create the OperatorGroup CR by running the following command: ++ +---- +$ oc create -f sriov-operatorgroup.yaml +---- + +.. Run the following command to get the `channel` value required for the next +step. ++ +---- +$ oc get packagemanifest sriov-network-operator -n openshift-marketplace -o jsonpath='{.status.channels[].name}' + +alpha +---- + +.. Create the following Subscription CR and save the YAML in the `sriov-sub.yaml` file: ++ +.Example Subscription +[source,yaml] +---- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: sriov-network-operator-subsription + namespace: sriov-network-operator +spec: + channel: <1> + name: sriov-network-operator + source: redhat-operators <2> + sourceNamespace: openshift-marketplace +---- +<1> Specify the value from you obtained in the previous step for the `.status.channels[].name` parameter. +<2> You must specify the `redhat-operators` value. + +.. Create the Subscription object by running the following command: ++ +---- +$ oc create -f sriov-sub.yaml +---- + +.. Change to the `sriov-network-operator` project: ++ +---- +$ oc project sriov-network-operator + +Now using project "sriov-network-operator" +---- + +[id="install-operator-web-console"] +== Installing the Operator using the web console + +As a cluster administrator, you can install the Operator using the web console. + +[NOTE] +==== +You have to create the Namespace CR and OperatorGroup CR as mentioned +in the previous section. +==== + +.Procedure + +. Install the SR-IOV Network Operator using the {product-title} web console: + +.. In the {product-title} web console, click *Operators* -> *OperatorHub*. + +.. Choose *SR-IOV Network Operator* from the list of available Operators, and then click *Install*. + +.. On the *Create Operator Subscription* page, under *A specific namespace on the cluster* select *sriov-network-operator*. Then, click *Subscribe*. + +. Optional: Verify that the SR-IOV Network Operator installed successfully: + +.. Switch to the *Operators* -> *Installed Operators* page. + +.. Ensure that *SR-IOV Network Operator* is listed in the *sriov-network-operator* project with a *Status* of *InstallSucceeded*. ++ +[NOTE] +==== +During installation an Operator might display a *Failed* status. +If the installation later succeeds with an *InstallSucceeded* message, you can ignore the *Failed* message. +==== + ++ +If the operator does not appear as installed, to troubleshoot further: + ++ +* Go to the *Operators* -> *Installed Operators* page and inspect +the *Operator Subscriptions* and *Install Plans* tabs for any failure or errors +under *Status*. +* Go to the *Workloads* -> *Pods* page and check the logs for Pods in the +`sriov-network-operator` project. + diff --git a/modules/nw-sriov-network-attachment.adoc b/modules/nw-sriov-network-attachment.adoc new file mode 100644 index 0000000000..a5dd9a24e9 --- /dev/null +++ b/modules/nw-sriov-network-attachment.adoc @@ -0,0 +1,57 @@ +// Module included in the following assemblies: +// +// * networking/multiple-networks/configuring-sr-iov.adoc + +[id="configuring-sr-iov-networks_{context}"] += Configuring SR-IOV additional network + +You can configure an additional network that uses SR-IOV hardware by creating a SriovNetwork Custom Resource (CR). +When you create a SriovNetwork CR, the SR-IOV Operator automatically creates a NetworkAttachmentDefinition CR. + +[NOTE] +===== +Do not modify or delete a SriovNetwork Custom Resource (CR) if it is attached to any Pods in the `running` state. +===== + +.Prerequisites + +* Install the OpenShift Command-line Interface (CLI), commonly known as `oc`. +* Log in as a user with `cluster-admin` privileges. + +.Procedure + +. Create the following SriovNetwork CR, and then save the YAML in the `-sriov-network.yaml` file. Replace `` with a name for this additional network. ++ +[source,yaml] +---- +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetwork +metadata: + name: <1> + namespace: sriov-network-operator <2> +spec: + networkNamespace: <3> + ipam: |- <4> + ... + vlan: <5> + resourceName: <6> +---- +<1> Replace `` with a name for the CR. The Operator will create a NetworkAttachmentDefinition CR with same name. +<2> Specify the namespace where the SR-IOV Operator is installed. +<3> Replace `` with the namespace where the NetworkAttachmentDefinition CR will be created. +<4> Specify a configuration object for the ipam CNI plug-in as a YAML block scalar. The plug-in manages IP address assignment for the attachment definition. +<5> Replace `` with a Virtual LAN (VLAN) ID for the additional network. The integer value must be from `0` to `4095`. The default value is `0`. +<6> Replace `` with the value for the `.spec.resourceName` parameter from the SriovNetworkNodePolicy CR that defines the SR-IOV hardware for this additional network. + +. Create the CR by running the following command: ++ +---- +$ oc create -f <1> +---- +<1> Replace `` with the name of the file you created in the previous step. + +. Optional: Confirm that the NetworkAttachmentDefinition CR associated with the SriovNetwork CR that you created in the previous step exists by running the following command. Replace `` with the namespace you specified in the SriovNetwork CR. ++ +---- +oc get net-attach-def -n +---- diff --git a/modules/nw-sriov-supported-devices.adoc b/modules/nw-sriov-supported-devices.adoc new file mode 100644 index 0000000000..2e7590fffe --- /dev/null +++ b/modules/nw-sriov-supported-devices.adoc @@ -0,0 +1,13 @@ +// Module included in the following assemblies: +// +// * networking/multiple-networks/configuring-sr-iov.adoc + +[id="supported-devices_{context}"] += Supported devices + +The following Network Interface Card (NIC) models are supported in +{product-title}: + +* Intel XXV710-DA2 25G card with vendor ID `0x8086` and device ID `0x158b` +* Mellanox MT27710 Family [ConnectX-4 Lx] 25G card with vendor ID `0x15b3` and device ID `0x1015` +* Mellanox MT27800 Family [ConnectX-5] 100G card with vendor ID `0x15b3` and device ID `0x1017` diff --git a/networking/multiple-networks/configuring-sr-iov.adoc b/networking/multiple-networks/configuring-sr-iov.adoc index 0187345ee6..1d129c2f83 100644 --- a/networking/multiple-networks/configuring-sr-iov.adoc +++ b/networking/multiple-networks/configuring-sr-iov.adoc @@ -5,8 +5,120 @@ include::modules/common-attributes.adoc[] toc::[] -:FeatureName: SR-IOV multinetwork support +:FeatureName: Network Interface Card (NIC) SR-IOV hardware include::modules/technology-preview.adoc[leveloffset=+1] -include::modules/nw-multinetwork-sriov.adoc[leveloffset=+1] +[id="about-using-sr-iov-hardware"] +== About SR-IOV hardware on {product-title} + +{product-title} includes the capability to use SR-IOV hardware on your nodes. +You can attach SR-IOV virtual function (VF) interfaces to Pods on nodes with SR-IOV hardware. + +You can use the {product-title} console to install SR-IOV by deploying the +SR-IOV Network Operator. The SR-IOV Network Operator creates and manages the +components of the SR-IOV stack. The Operator provides following features: + +* Discover the SR-IOV network device in cluster. +* Initialize the supported SR-IOV NIC models on nodes. +* Provision the SR-IOV network device plug-in on nodes. +* Provision the SR-IOV CNI plug-in executable on nodes. +* Provision the Network Resources Injector in cluster. +* Manage configuration of SR-IOV network device plug-in. +* Generate NetworkAttachmentDefinition custom resources (CR) for the SR-IOV CNI +plug-in. + +Here's the function of each above mentioned SR-IOV components. + +* The SR-IOV network device plug-in is a Kubernetes device plug-in for +discovering, advertising, and allocating SR-IOV network virtual function (VF) +resources. Device plug-ins are used in Kubernetes to enable the use of limited +resources, typically in physical devices. Device plug-ins give the Kubernetes +scheduler awareness of resource availability, so the scheduler can schedule Pods on nodes with sufficient resources. + +* The SR-IOV CNI plug-in plumbs VF interfaces allocated from the SR-IOV device +plug-in directly into a Pod. + +* The Network Resources Injector is a Kubernetes Dynamic Admission Controller +Webhook that provides functionality for patching Kubernetes Pod specifications +with requests and limits for custom network resources such as SR-IOV VFs. + +[NOTE] +==== +The Network Resources Injector is enabled by default and cannot be disabled. +==== + +include::modules/nw-sriov-supported-devices.adoc[leveloffset=+2] + +include::modules/nw-sriov-device-discovery.adoc[leveloffset=+2] + +[id="example-vf-use-in-pod"] +=== Example use of virtual function (VF) in a Pod + +You can run a remote direct memory access (RDMA) or a Data Plane Development Kit (DPDK) application in a Pod with SR-IOV VF attached. +In the following example, a Pod is using a VF in RDMA mode: + +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: rdma-app + annotations: + k8s.v1.cni.cncf.io/networks: sriov-rdma-mlnx +spec: + containers: + - name: testpmd + image: + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + add: ["IPC_LOCK"] + command: ["sleep", "infinity"] +---- + +The following example shows a Pod with VF in DPDK mode: + +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: dpdk-app + annotations: + k8s.v1.cni.cncf.io/networks: sriov-dpdk-net +spec: + containers: + - name: testpmd + image: + securityContext: + capabilities: + add: ["IPC_LOCK"] + volumeMounts: + - mountPath: /dev/hugepages + name: hugepage + resources: + limits: + memory: "1Gi" + cpu: "2" + hugepages-1Gi: "4Gi" + requests: + memory: "1Gi" + cpu: "2" + hugepages-1Gi: "4Gi" + command: ["sleep", "infinity"] + volumes: + - name: hugepage + emptyDir: + medium: HugePages +---- + +include::modules/nw-sriov-installing-operator.adoc[leveloffset=+1] + +include::modules/nw-sriov-configuring-device.adoc[leveloffset=+1] + +include::modules/nw-sriov-network-attachment.adoc[leveloffset=+1] + +include::modules/nw-multus-ipam-object.adoc[leveloffset=+2] + +include::modules/nw-multus-add-pod.adoc[leveloffset=+1]