diff --git a/_topic_maps/_topic_map_ms.yml b/_topic_maps/_topic_map_ms.yml index 3414f596df..426045d003 100644 --- a/_topic_maps/_topic_map_ms.yml +++ b/_topic_maps/_topic_map_ms.yml @@ -424,6 +424,8 @@ Topics: Topics: - Name: About using multiple networks File: microshift-cni-multus + - Name: Configuring and using multiple networks + File: microshift-cni-multus-using - Name: Firewall configuration File: microshift-firewall - Name: Networking settings for fully disconnected hosts diff --git a/microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc b/microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc new file mode 100644 index 0000000000..7b2076f5b6 --- /dev/null +++ b/microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc @@ -0,0 +1,29 @@ +:_mod-docs-content-type: ASSEMBLY +[id="microshift-cni-multus-using"] += Configuring and using multiple networks +include::_attributes/attributes-microshift.adoc[] +:context: microshift-cni-multus-using + +toc::[] + +After you have installed the {microshift-short} Multus Container Network Interface (CNI), you can use other networking plugins by using configurations. + +include::modules/microshift-cni-multus-using-ipam.adoc[leveloffset=+1] + +include::modules/microshift-cni-multus-nad-additional-network.adoc[leveloffset=+1] + +//MicroShift-edited version of OCP procedure +include::modules/microshift-nw-multus-add-pod.adoc[leveloffset=+1] + +include::modules/microshift-cni-multus-add-network-example-config.adoc[leveloffset=+1] + +//OCP procedure, edit with conditionals and care +include::modules/nw-multus-remove-pod.adoc[leveloffset=+1] + +include::modules/microshift-cni-multus-troubleshoot.adoc[leveloffset=+1] + +[id="additional-resources_microshift-cni-multus-using_{context}"] +== Additional resources +* xref:../../microshift_networking/microshift_multiple_networks/microshift-cni-multus.adoc#microshift-cni-multus[About using multiple networks] + +* link:https://access.redhat.com/documentation/en-us/openshift_container_platform/4.15/html/networking/multiple-networks#nw-multus-ipam-object_configuring-additional-network[Configuration of IP address assignment for an additional network] \ No newline at end of file diff --git a/microshift_networking/microshift_multiple_networks/microshift-cni-multus.adoc b/microshift_networking/microshift_multiple_networks/microshift-cni-multus.adoc index d33f4eabe9..6c99de8c01 100644 --- a/microshift_networking/microshift_multiple_networks/microshift-cni-multus.adoc +++ b/microshift_networking/microshift_multiple_networks/microshift-cni-multus.adoc @@ -14,7 +14,6 @@ include::modules/microshift-install-multus-running-cluster.adoc[leveloffset=+1] //OCP module, edit with conditionals and care include::modules/nw-multus-bridge-object.adoc[leveloffset=+1] -//Q: do we need to remove VLAN from this reference for microshift? //OCP module, edit with conditionals and care include::modules/nw-multus-ipvlan-object.adoc[leveloffset=+1] @@ -22,7 +21,7 @@ include::modules/nw-multus-ipvlan-object.adoc[leveloffset=+1] //OCP module, edit with conditionals and care include::modules/nw-multus-macvlan-object.adoc[leveloffset=+1] -[role="_additional-resources"] -.Additional resources +[id="additional-resources_microshift-cni-multus_{context}"] +== Additional resources -//* add cross refs to procedures +* xref:../../microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc#microshift-cni-multus-using[Configuring and using multiple networks] diff --git a/modules/microshift-cni-multus-add-network-example-config.adoc b/modules/microshift-cni-multus-add-network-example-config.adoc new file mode 100644 index 0000000000..f710b30106 --- /dev/null +++ b/modules/microshift-cni-multus-add-network-example-config.adoc @@ -0,0 +1,129 @@ +// Module included in the following assemblies: +// +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-cni-multus-add-network-example-config_{context}"] += Configuring an additional network + +After you have created the NetworkAttachmentDefinition object and applied it, use the following example procedure to configure an additional network. In this example, the `bridge` type additional network is used. You can also use this workflow for other additional network types. + +.Prerequisite +. You created and applied the `NetworkAttachmentDefinition` object configuration. + +.Procedure +. Verify that the bridge was created on the host by running the following command: ++ +[source,terminal] +---- +$ ip a show br-test +---- ++ +.Example output +[source,terminal] +---- +22: br-test: mtu 1500 qdisc noqueue state UP group default qlen 1000 + link/ether 96:bf:ca:be:1d:15 brd ff:ff:ff:ff:ff:ff + inet6 fe80::34e2:bbff:fed2:31f2/64 scope link + valid_lft forever preferred_lft forever +---- + +. Configure an IP address for the bridge by running the following command: ++ +[source,terminal] +---- +$ sudo ip addr add 10.10.0.10/24 dev br-test +---- + +. Verify that the IP address configuration is added to the bridge by running the following command: ++ +[source,terminal] +---- +$ ip a show br-test +---- ++ +.Example output +[source,terminal] +---- +22: br-test: mtu 1500 qdisc noqueue state UP group default qlen 1000 + link/ether 96:bf:ca:be:1d:15 brd ff:ff:ff:ff:ff:ff + inet 10.10.0.10/24 scope global br-test <1> + valid_lft forever preferred_lft forever + inet6 fe80::34e2:bbff:fed2:31f2/64 scope link + valid_lft forever preferred_lft forever +---- +<1> The IP address is configured as expected. + +. Verify the IP address of the pod by running the following command: ++ +[source,terminal] +---- +$ oc get pod test-bridge --output=jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' +---- ++ +.Example output +[source,terminal] +---- +[{ + "name": "ovn-kubernetes", + "interface": "eth0", + "ips": [ + "10.42.0.17" + ], + "mac": "0a:58:0a:2a:00:11", + "default": true, + "dns": {} +},{ + "name": "default/bridge-conf", <1> + "interface": "net1", + "ips": [ + "10.10.0.20" + ], + "mac": "82:01:98:e5:0c:b7", + "dns": {} +---- +<1> The bridge additional network is attached as expected. + +. Optional: You can use `oc exec` to access the pod and confirm its interfaces by using the `ip` command: ++ +[source,terminal] +---- +$ oc exec -ti test-bridge -- ip a +---- ++ +.Example output +[source,terminal] +---- +1: lo: mtu 65536 qdisc noqueue qlen 1000 + link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 + inet 127.0.0.1/8 scope host lo + valid_lft forever preferred_lft forever + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever +2: eth0@if21: mtu 1500 qdisc noqueue + link/ether 0a:58:0a:2a:00:11 brd ff:ff:ff:ff:ff:ff + inet 10.42.0.17/24 brd 10.42.0.255 scope global eth0 + valid_lft forever preferred_lft forever + inet6 fe80::858:aff:fe2a:11/64 scope link + valid_lft forever preferred_lft forever +3: net1@if23: mtu 1500 qdisc noqueue + link/ether 82:01:98:e5:0c:b7 brd ff:ff:ff:ff:ff:ff + inet 10.10.0.20/24 brd 10.10.0.255 scope global net1 <1> + valid_lft forever preferred_lft forever + inet6 fe80::8001:98ff:fee5:cb7/64 scope link + valid_lft forever preferred_lft forever +---- +<1> Pod is attached to the 10.10.0.20 IP address on the `net1 interface` as expected. + +. Confirm that the connection is working as expected by accessing the HTTP server in the pod from the {microshift-short} host. Use the following command: ++ +[source,terminal] +---- +$ curl 10.10.0.20:8080 +---- ++ +.Example output +[source,terminal] +---- +Hello MicroShift +---- diff --git a/modules/microshift-cni-multus-nad-additional-network.adoc b/modules/microshift-cni-multus-nad-additional-network.adoc new file mode 100644 index 0000000000..6d2adee4a4 --- /dev/null +++ b/modules/microshift-cni-multus-nad-additional-network.adoc @@ -0,0 +1,89 @@ +// Module included in the following assemblies: +// +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-cni-multus-nad-additional-network_{context}"] += Creating a NetworkAttachmentDefinition for an additional network + +Use the following procedure to create a `NetworkAttachmentDefinition` configuration file for an additional network. In this example, a bridge-type interface is used. You can also use the example workflow here that uses `host-local` IP address management (IPAM) to configure other supported additional network types. + +[IMPORTANT] +==== +If you use `bridge` and the `dhcp` IPAM, a DHCP server listening on the bridged network is required. If you are also using a firewall, configuring the firewalld service to allow DHCP traffic on the network zone is also required. You can run the `firewall-cmd --remove-service=dhcp` command in this case. +==== + +.Prerequisites + +* The {microshift-short} Multus CNI is installed. +* The OpenShift CLI (`oc`) is installed. +* The cluster is running. + +.Procedure + +. Optional: Verify that the {microshift-short} cluster is running with the Multus CNI by running the following command: ++ +[source,terminal] +---- +$ oc get pods -n openshift-multus +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +dhcp-daemon-dfbzw 1/1 Running 0 5h +multus-rz8xc 1/1 Running 0 5h +---- + +. Create a `NetworkAttachmentDefinition` configuration file by running the following command and using the following example file for reference: ++ +[source,terminal] +---- +$ oc apply -f network-attachment-definition.yaml +---- ++ +.Example `NetworkAttachmentDefinition` file +[source,yaml] +---- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: bridge-conf +spec: + config: '{ + "cniVersion": "0.4.0", + "type": "bridge", <1> + "bridge": "br-test", <2> + "mode": "bridge", + "ipam": { + "type": "host-local", <3> + "ranges": [ + [ + { + "subnet": "10.10.0.0/24", + "rangeStart": "10.10.0.20", + "rangeEnd": "10.10.0.50", + "gateway": "10.10.0.254" + } + ], + [ + { + "subnet": "fd00:IJKL:MNOP:10::0/64", <4> + "rangeStart": "fd00:IJKL:MNOP:10::1", + "rangeEnd": "fd00:IJKL:MNOP:10::9" + "dataDir": "/var/lib/cni/br-test" + } + }' +---- +<1> The `type` value specifies a name of the CNI plugin. This example uses the `bridge` type. +<2> The `bridge` value is name of the bridge on the {microshift-short} host that is used. The additional interface of the pod is connected to that bridge. If the interface does not exist on the host, the Bridge CNI creates it. If the interface already exists, it is reused. In this example, the name of the interface is `br-test`. +<3> The IPAM type. +<4> IPv6 addresses can be added to the secondary interface. ++ +-- +[NOTE] +==== +Using the name of the bridge is specific to the `bridge` type of plugin. Other plugins use different fields in their `NetworkAttachmentDefinitions`. For example, the `macvlan` and `ipvlan` configurations use `master` to specify the host interface to attach. +==== +-- \ No newline at end of file diff --git a/modules/microshift-cni-multus-troubleshoot.adoc b/modules/microshift-cni-multus-troubleshoot.adoc new file mode 100644 index 0000000000..ea252d6f85 --- /dev/null +++ b/modules/microshift-cni-multus-troubleshoot.adoc @@ -0,0 +1,52 @@ +// Module included in the following assemblies: +// +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + +:_mod-docs-content-type: REFERENCE +[id="microshift-cni-multus-troubleshoot_{context}"] += Troubleshooting Multus networking + +If the settings for multiple networks are not configured properly, pods can fail to start. The following steps can help you solve for a couple common scenarios. + +[id="Pod-networking-cannot-be-configured_{context}"] +== Pod networking cannot be configured + +If the Multus CNI plugin cannot apply networking annotations to a pod, the pod does not start. Pods can also fail to start if any of the additional network CNIs fail. + +.Example error +[source,terminal] +---- +Warning NoNetworkFound 0s multus cannot find a network-attachment-definitio (asdasd) in namespace (default): network-attachment-definitions.k8s.cni.cncf.io "bad-ref-doesnt-exist" not found +---- + +In this case, you can take the following steps to trouble CNI failures: + +* Verify the values in both the `NetworkAttachmentDefinitions` and the annotations. +* Remove the annotation to verify whether the pod is created successfully with just the default network. If not, this might indicate a networking problem other than the Multus configuration. +* If you are a device administrator, you can inspect the `crio.service` or `microshift.service` logs, paying special attention to those that are generated by the `kubelet`. ++ +For example, the following error from the `kubelet` shows that the primary CNI is not running. This situation can be caused by pods not starting or because of a CRI-O misconfiguration such as an incorrect `cni_default_network` setting. ++ +.Example kubelet-generated error +[source,terminal] +---- +Feb 06 13:47:31 dev microshift[1494]: kubelet E0206 13:47:31.163290 1494 pod_workers.go:1298] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: No CNI configuration file in /etc/cni/net.d/. Has your network provider started?" pod="default/samplepod" podUID="fe0f7f7a-8c47-4488-952b-8abc0d8e2602" +---- + +[id="missing-nad_{context}"] +== Missing configuration file + +Sometimes a pod cannot be created because the annotations reference a `NetworkAttachmentDefinition` configuration YAML that does not exist. In this case an error such as the following is usually produced: + +.Example log +[source,terminal] +---- +cannot find a network-attachment-definition (bad-conf) in namespace (default): network-attachment-definitions.k8s.cni.cncf.io "bad-conf" not found" pod="default/samplepod"` +---- +.Example error output +[source,terminal] +---- +"CreatePodSandbox for pod failed" err="rpc error: code = Unknown desc = failed to create pod network sandbox k8s_samplepod_default_5fa13105-1bfb-4c6b-aee7-3437cfb50e25_0(7517818bd8e85f07b551f749c7529be88b4e7daef0dd572d049aa636950c76c6): error adding pod default_samplepod to CNI network \"multus-cni-network\": plugin type=\"multus\" name=\"multus-cni-network\" failed (add): Multus: [default/samplepod/5fa13105-1bfb-4c6b-aee7-3437cfb50e25]: error loading k8s delegates k8s args: TryLoadPodDelegates: error in getting k8s network for pod: GetNetworkDelegates: failed getting the delegate: getKubernetesDelegate: cannot find a network-attachment-definition (bad-conf) in namespace (default): network-attachment-definitions.k8s.cni.cncf.io \"bad-conf\" not found" pod="default/samplepod" +---- + +To fix this error, create and apply the `NetworkAttachmentDefinitions` YAML. \ No newline at end of file diff --git a/modules/microshift-cni-multus-using-ipam.adoc b/modules/microshift-cni-multus-using-ipam.adoc new file mode 100644 index 0000000000..6951f723ac --- /dev/null +++ b/modules/microshift-cni-multus-using-ipam.adoc @@ -0,0 +1,21 @@ +// Module included in the following assemblies: +// +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + +:_mod-docs-content-type: CONCEPT +[id="IP-address-management-types-and-additional-networks_{context}"] += IP address management types and additional networks + +IP addresses are provisioned for an additional network through an IP Address Management (IPAM) CNI plugin that you configure. Supported IP address provisioning types in {microshift-short} are `host-local`, `static`, and `dhcp`. + +[id="bridge-interface-specifics_{context}"] +== bridge interface specifics +When using the `bridge` type interface and the `dhcp` IPAM, a DHCP server listening on the bridged network is required. If you are using a firewall, configuring the `firewalld` service by running the `firewall-cmd --remove-service=dhcp` command to allow DHCP traffic on the network zone is also required. + +[id="macvlan-interface-specifics_{context}"] +== macvlan interface specifics +The `macvlan` type interface accesses the network that the host is connected to. This means that the interface can receive an IP address from the DHCP server on the host network if the `dhcp` IPAM plugin is used. + +[id="ipvlan-interface-specifics_{context}"] +== ipvlan interface specifics +The `ipvlan` interface also has direct access to the host network, but shares a MAC address with the host interface. The `ipvlan` type interface cannot be used with the `dhcp` plugin because of the shared MAC address. The IPAM plugin does not support the DHCP protocol with `ClientID`. \ No newline at end of file diff --git a/modules/microshift-nw-multus-add-pod.adoc b/modules/microshift-nw-multus-add-pod.adoc new file mode 100644 index 0000000000..6ce10592a9 --- /dev/null +++ b/modules/microshift-nw-multus-add-pod.adoc @@ -0,0 +1,193 @@ +// Module included in the following assemblies: +// +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + +:_mod-docs-content-type: PROCEDURE +[id="microshift-nw-multus-add-pod_{context}"] += Adding a pod to an additional network + +You can add a pod to an additional network. At the time a pod is created, additional networks are attached to it. The pod continues to send normal cluster-related network traffic over the default network. + +If you want to attach additional networks to a pod that is already running, you must restart the pod. + +.Prerequisites + +* The OpenShift CLI (`oc`) is installed. +* The cluster is running. +* A network defined by a `NetworkAttachmentDefinition` object that you want to attach the pod to exists. + +.Procedure + +. Add an annotation to a `Pod` YAML file. Only one of the following annotation formats can be used: + +.. To attach an additional network without any customization, add an annotation with the following format. Replace `` with the name of the additional network to associate with the pod: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: [,,...] <1> +# ... +---- +<1> Replace `` with the name of the additional network to associate with the pod. To specify more than one additional network, separate each network with a comma. Do not include whitespace between the comma. If you specify the same additional network multiple times, that pod will have multiple network interfaces attached to that network. ++ +.Example annotation for a bridge-type additional network ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: bridge-conf +# ... +---- + +.. To attach an additional network with customizations, add an annotation with the following format: ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: |- + [ + { + "name": "", <1> + "namespace": "", <2> + "default-route": [""] <3> + } + ] +# ... +---- +<1> Specify the name of the additional network defined by a `NetworkAttachmentDefinition` object. +<2> Specify the namespace where the `NetworkAttachmentDefinition` object is defined. +<3> Optional: Specify an override for the default route, such as `192.168.17.1`. + +. To create a `Pod` YAML file and add the `NetworkAttachmentDefinition` annotation for an additional network, run the following command and use the example YAML: ++ +[source,terminal] +---- +$ oc apply -f ./.yaml <1> +---- +<1> Replace __ with the pod name that you want to use. ++ +.Example output +[source,terminal] +---- +pod/test-bridge created +---- ++ +.Example `test-bridge` pod YAML +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + name: test-bridge + annotations: + k8s.v1.cni.cncf.io/networks: bridge-conf + labels: + app: test-bridge +spec: + terminationGracePeriodSeconds: 0 + containers: + - name: hello-microshift + image: quay.io/microshift/busybox:1.36 + command: ["/bin/sh"] + args: ["-c", "while true; do echo -ne \"HTTP/1.0 200 OK\r\nContent-Length: 16\r\n\r\nHello MicroShift\" | nc -l -p 8080 ; done"] + ports: + - containerPort: 8080 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + seccompProfile: + type: RuntimeDefault +---- ++ +. Make sure that the `NetworkAttachmentDefinition` annotation is correct: ++ +.Example `NetworkAttachmentDefinition` annotation ++ +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: bridge-conf +# ... +---- + +. Optional: To confirm that the `NetworkAttachmentDefinition` annotation exists in a `Pod` YAML, run the following command, replacing `` with the name of the pod. ++ +[source,terminal] +---- +$ oc get pod -o yaml <1> +---- +<1> Replace __ with the pod name you want to use. In the following example, `test-bridge` is used. ++ +In the following example, the `test-bridge` is attached to the `net1` additional network: ++ +[source,terminal] +---- +$ oc get pod test-bridge -o yaml +---- ++ +.Example output +[source,yaml] +---- +apiVersion: v1 +kind: Pod +metadata: + annotations: + k8s.v1.cni.cncf.io/networks: bridge-conf + k8s.v1.cni.cncf.io/network-status: |- <1> + [{ + "name": "ovn-kubernetes", + "interface": "eth0", + "ips": [ + "10.42.0.18" + ], + "default": true, + "dns": {} + },{ + "name": "bridge-conf", + "interface": "net1", + "ips": [ + "20.2.2.100" + ], + "mac": "22:2f:60:a5:f8:00", + "dns": {} + }] + name: pod + namespace: default +spec: +# ... +status: +# ... +---- +<1> The `k8s.v1.cni.cncf.io/network-status` parameter is a JSON array of objects. Each object describes the status of an additional network attached to the pod. The annotation value is stored as a plain text value. + +. Verify that the pod is running by running the following command: ++ +[source,terminal] +---- +$ oc get pod +---- ++ +.Example output +[source,terminal] +---- +NAME READY STATUS RESTARTS AGE +test-bridge 1/1 Running 0 81s +---- diff --git a/modules/nw-multus-remove-pod.adoc b/modules/nw-multus-remove-pod.adoc index b22b5157c3..5be661515c 100644 --- a/modules/nw-multus-remove-pod.adoc +++ b/modules/nw-multus-remove-pod.adoc @@ -1,6 +1,8 @@ // Module included in the following assemblies: // // * networking/multiple_networks/removing-pod.adoc +// * microshift_networking/microshift_multiple_networks/microshift-cni-multus-using.adoc + :_mod-docs-content-type: PROCEDURE [id="nw-multus-remove-pod_{context}"]