Add performance code to set ovs

defaults/main.yml

@@ -2,3 +2,4 @@
 configure_kernel: true
 bind_drivers: true
 set_ovs: true
+pmd_threads_count: 1

library/configure_kernel.py

@@ -51,6 +51,8 @@ import traceback
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.dpdk_setup_common import exec_cmd
+from ansible.module_utils.dpdk_setup_common import get_cpu_list
+from ansible.module_utils.dpdk_setup_common import DPDK_DRIVERS
 
 
 class ReadKernelArgsError(Exception):
@@ -65,45 +67,6 @@ class SelectCpuPartitioningError(Exception):
     pass
 
 
-def _get_cpu_list(pci_addresses):
-    cores = []
-    for addr in pci_addresses:
-        local_cpu_list = _get_nic_cpus_without_zero_core(addr)
-        if local_cpu_list not in cores:
-            cores.append(local_cpu_list)
-    return ','.join(cores)
-
-
-def _get_nic_cpus_without_zero_core(pci_address):
-    local_cpu_list = _get_nic_cpu_list(pci_address)
-    if _is_first_core_zero(local_cpu_list):
-        local_cpu_list = _remove_first_core(local_cpu_list)
-    return local_cpu_list
-
-
-def _get_nic_cpu_list(pci_address):
-    DEVICE_PATH_FMT = '/sys/bus/pci/devices/{}'
-
-    local_cpulist = os.path.join(
-        DEVICE_PATH_FMT.format(pci_address), 'local_cpulist'
-    )
-    with open(local_cpulist) as f:
-        return f.read()
-
-
-def _is_first_core_zero(cores):
-    return cores[:1] == '0'
-
-
-def _remove_first_core(cores):
-    if cores[1] == '-':
-        return '1' + cores[1:]
-    elif cores[1] == ',':
-        return cores[2:]
-    else:
-        return ""
-
-
 def _get_default_kernel():
     proc = subprocess.Popen(['grubby', '--default-kernel'],
                             stdout=subprocess.PIPE)
@@ -194,7 +157,7 @@ def _is_iommu_set():
 
 
 def _configure_kernel(pci_addresses):
-    cpu_list = _get_cpu_list(pci_addresses)
+    cpu_list = get_cpu_list(pci_addresses)
     default_kernel = _get_default_kernel()
 
     added_hugepages = _add_hugepages(default_kernel)
@@ -206,8 +169,6 @@ def _configure_kernel(pci_addresses):
 
 
 def main():
-    DPDK_DRIVERS = ('vfio-pci',)
-
     module = AnsibleModule(
         argument_spec=dict(
             pci_drivers=dict(default=None, type='dict', required=True)

library/set_ovs_dpdk_facts.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import subprocess
+import traceback
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.dpdk_setup_common import \
+    get_nic_cpus_without_zero_core
+from ansible.module_utils.dpdk_setup_common import get_cpu_list
+from ansible.module_utils.dpdk_setup_common import DEVICE_PATH_FMT
+from ansible.module_utils.dpdk_setup_common import DPDK_DRIVERS
+
+
+def get_dpdk_nics_numa_info(pci_addresses):
+    nics_per_numa = {}
+    for addr in pci_addresses:
+        numa_node = int(_get_numa_node(addr))
+        if numa_node == -1:
+            numa_node = 0
+        if numa_node in nics_per_numa:
+            nics_per_numa[numa_node]['nics'] += 1
+        else:
+            nics_per_numa[numa_node] = {}
+            nics_per_numa[numa_node]['nics'] = 1
+            nics_per_numa[numa_node]['cpu_list'] = \
+                get_nic_cpus_without_zero_core(addr)
+
+    return nics_per_numa
+
+
+def _get_numa_node(pci_address):
+    numa_node = os.path.join(
+        DEVICE_PATH_FMT.format(pci_address),
+        'numa_node'
+    )
+    with open(numa_node) as f:
+        return f.read()
+
+
+def _range_to_list(core_list):
+    edges = core_list.rstrip().split('-')
+    return list(range(int(edges[0]), int(edges[1]) + 1))
+
+
+def _list_from_string(str_list):
+    return list(map(int, str_list.rstrip().split(',')))
+
+
+def _get_cpu_list(cores):
+    if '-' in cores:
+        return _range_to_list(cores)
+    if ',' in cores:
+        return _list_from_string(cores)
+
+
+def _get_numa_nodes_nr():
+    ls_proc = subprocess.Popen(
+            "ls -l /sys/devices/system/node/".split(),
+            stdout=subprocess.PIPE)
+    grep_proc = subprocess.Popen(
+            "grep node".split(),
+            stdin=ls_proc.stdout,
+            stdout=subprocess.PIPE)
+    wc_proc = subprocess.Popen(
+            "wc -l".split(),
+            stdin=grep_proc.stdout,
+            stdout=subprocess.PIPE)
+
+    output, error = wc_proc.communicate()
+    return int(output)
+
+
+def get_core_mask(cores):
+    mask = 0
+    for core in cores:
+        mask |= (1 << int(core))
+    return hex(mask)
+
+
+def get_pmd_cores(nics_numa_info, pmd_threads_count):
+    pmd_cores = []
+    for node_info in nics_numa_info.values():
+        nics_count = node_info['nics']
+        cores = _get_cpu_list(node_info['cpu_list'])
+
+        num_cores = nics_count * pmd_threads_count
+        while num_cores > 0:
+            min_core = min(cores)
+            pmd_cores.append(min_core)
+            cores.remove(min_core)
+            num_cores -= 1
+
+    return pmd_cores
+
+
+def get_dpdk_lcores(pmd_cores, cpu_list):
+    socket_mem = ""
+    cores = _get_cpu_list(cpu_list)
+    available_cores = list(set(cores) - set(pmd_cores))
+    return available_cores[:2]
+
+
+def get_socket_mem(nics_numa_info):
+    socket_mem_list = []
+    numa_nodes = list(nics_numa_info.keys())
+
+    for i in range(0, _get_numa_nodes_nr()):
+        if i in numa_nodes:
+            socket_mem_list.append('2048')
+        else:
+            socket_mem_list.append('1024')
+
+    return ','.join(socket_mem_list)
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            pci_drivers=dict(default=None, type='dict', required=True),
+            pmd_threads_count=dict(default=None, type='int', required=True)
+        )
+    )
+    pci_drivers = module.params.get('pci_drivers')
+    pci_addresses = [addr for addr, driver in pci_drivers.iteritems()
+                     if driver in DPDK_DRIVERS]
+    pmd_threads_count = module.params.get('pmd_threads_count')
+    try:
+        dpdk_nics_numa_info = get_dpdk_nics_numa_info(pci_addresses)
+
+        socket_mem = get_socket_mem(dpdk_nics_numa_info)
+
+        pmd_cores = get_pmd_cores(dpdk_nics_numa_info, pmd_threads_count)
+        pmd_cpu_mask = get_core_mask(pmd_cores)
+
+        dpdk_lcores = get_dpdk_lcores(pmd_cores, get_cpu_list(pci_addresses))
+        dpdk_lcores_mask = get_core_mask(dpdk_lcores)
+    except Exception as e:
+        module.fail_json(msg=str(e), exception=traceback.format_exc())
+
+    module.exit_json(
+        dpdk_socket_mem=socket_mem,
+        pmd_cpu_mask=pmd_cpu_mask,
+        dpdk_lcores_mask=dpdk_lcores_mask
+    )
+
+if __name__ == "__main__":
+    main()

module_utils/dpdk_setup_common.py

@@ -16,12 +16,54 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
+import os
 import subprocess
 
 
+DEVICE_PATH_FMT = '/sys/bus/pci/devices/{}'
+DPDK_DRIVERS = ('vfio-pci',)
+
+
 def exec_cmd(args):
     proc = subprocess.Popen(
         args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     )
     out, err = proc.communicate()
     return proc.returncode, out, err
+
+
+def get_cpu_list(pci_addresses):
+    cores = []
+    for addr in pci_addresses:
+        local_cpu_list = get_nic_cpus_without_zero_core(addr)
+        if local_cpu_list not in cores:
+            cores.append(local_cpu_list)
+    return ','.join(cores)
+
+
+def get_nic_cpus_without_zero_core(pci_address):
+    local_cpu_list = _get_nic_cpu_list(pci_address)
+    if _is_first_core_zero(local_cpu_list):
+        local_cpu_list = _remove_first_core(local_cpu_list)
+    return local_cpu_list
+
+
+def _get_nic_cpu_list(pci_address):
+    local_cpulist = os.path.join(
+        DEVICE_PATH_FMT.format(pci_address), 'local_cpulist'
+    )
+    with open(local_cpulist) as f:
+        return f.read()
+
+
+def _is_first_core_zero(cores):
+    return cores[:1] == '0'
+
+
+def _remove_first_core(cores):
+    if cores[1] == '-':
+        return '1' + cores[1:]
+    elif cores[1] == ',':
+        return cores[2:]
+    else:
+        return ""

tasks/main.yml

@@ -1,27 +1,9 @@
-- name: configure kernel
-  configure_kernel:
-    pci_drivers: "{{ pci_drivers }}"
+---
+- include_tasks: configure_kernel.yml
   when: configure_kernel == true
 
-- name: bind devices to drivers
-  bind_drivers:
-    pci_drivers: "{{ pci_drivers }}"
+- include_tasks: bind_drivers.yml
   when: bind_drivers == true
 
-- block:
-  - name: install openvswitch
-    package:
-      name: "openvswitch"
-      state: installed
-
-  - name: start ovs
-    service:
-      name: openvswitch
-      state: started
-
-  - name: set dpdk-socket-mem
-    command: "ovs-vsctl --no-wait set open_vswitch . other_config:dpdk-socket-mem=512"
-
-  - name: set dpdk init
-    command: "ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true"
-  when: set_ovs == true
+- include_tasks: set_ovs_dpdk.yml
+  when: set_ovs_dpdk == true

tasks/set_ovs_dpdk.yml

@@ -0,0 +1,47 @@
+- name: populate ovs dpdk info
+  set_ovs_dpdk_facts:
+    pci_drivers: "{{ pci_drivers }}"
+    pmd_threads_count: "{{ pmd_threads_count }}"
+  register: ovs_facts
+
+- name: install openvswitch
+  package:
+    name: "openvswitch"
+    state: installed
+
+- name: set dpdk-init
+  openvswitch_db:
+    table: open_vswitch
+    record: .
+    col: other_config
+    key: dpdk-init
+    value: true
+
+- name: set pmd-cpu-mask
+  openvswitch_db:
+    table: open_vswitch
+    record: .
+    col: other_config
+    key: pmd-cpu-mask
+    value: "{{ ovs_facts.pmd_cpu_mask }}"
+
+- name: set dpdk lcore mask
+  openvswitch_db:
+    table: open_vswitch
+    record: .
+    col: other_config
+    key: dpdk-lcore-mask
+    value: "{{ ovs_facts.dpdk_lcores_mask }}"
+
+- name: set dpdk lcore mask
+  openvswitch_db:
+    table: open_vswitch
+    record: .
+    col: other_config
+    key: dpdk-socket-mem
+    value: "{{ ovs_facts.dpdk_socket_mem }}"
+
+- name: restart openvswitch service
+  service:
+    name: openvswitch
+    state: restarted