bpf-map-pinning: update implementation

Signed-off-by: Maryam Tahhan <mtahhan@redhat.com>
CloudNativeDataPlane · Mar 27, 2023 · a6c7532 · a6c7532
1 parent 652fd5d
commit a6c7532
Show file tree

Hide file tree

Showing 13 changed files with 67 additions and 41 deletions.
diff --git a/ansible/cndp-config/templates/fwd.jsonc.j2 b/ansible/cndp-config/templates/fwd.jsonc.j2
@@ -130,7 +130,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
-    //   xsk_pin_path - (O) Path to pinned bpf map
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

diff --git a/containerization/docker/ubuntu/Dockerfile b/containerization/docker/ubuntu/Dockerfile
@@ -88,6 +88,7 @@ RUN apt-get update && apt-get install -y \
 
 # Copy artifacts from the build container
 COPY --from=build /cndp/usr/local/bin/cndpfwd /usr/bin/
+COPY --from=build /cndp/usr/local/bin/cnet-graph /usr/bin/
 COPY --from=build /cndp/usr/local/bin/fwd /usr/bin
 COPY --from=build /cndp/usr/local/lib/x86_64-linux-gnu/*.so /usr/lib/
 COPY --from=build /cndp/lang/go/stats/prometheus/prometheus /usr/bin/

diff --git a/containerization/docker/ubuntu/fwd.jsonc b/containerization/docker/ubuntu/fwd.jsonc
@@ -115,7 +115,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
-    //   xsk_pin_path - (O) Path to pinned bpf map
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

diff --git a/containerization/k8s/README.md b/containerization/k8s/README.md
@@ -254,16 +254,17 @@ index 8465a21..774c6ca 100644
  data:
    config.json: |
      {
-         "mode": "cndp",
-         "logLevel": "debug",
-         "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
-         "pools" : [
-             {
--                "name" : "e2e",
-+                "name" : "pool1",
-                 "drivers" : ["i40e"]
-             }
-         ]
+		"clusterType":"physical",
+        "mode": "primary",
+        "logLevel": "debug",
+        "logFile": "/var/log/afxdp-k8s-plugins/cndp-dp-e2e.log",
+        "pools" : [
+            {
+-              "name" : "e2e",
++              "name" : "pool1",
+               "drivers" : ["i40e"]
+            }
+        ]
 
 ```
 
@@ -341,6 +342,10 @@ file to define LIST_OF_QIDS or CNDP_COPY_MODE environment variables. These
 variables are read by `tools/jsonc_gen.sh` when the container starts to
 generate the configuration for the CNDP application.
 
+> **_NOTE:_** the `tools/jsonc_gen.sh` supports BPF map pinning configuration by
+running it with the `-p` flag. If you wish to use it in a kind cluster then run
+the script with a `-k` flag in the pod YAML.
+
 An example to force copy-mode for all AF_XDP sockets:
 
 ```yaml

diff --git a/containerization/k8s/cndp-pods/cndp-0-0.yaml b/containerization/k8s/cndp-pods/cndp-0-0.yaml
@@ -16,12 +16,15 @@ spec:
     - name: cndp-0
       command: ["/bin/bash"]
       args: ["-c", "./jsonc_gen.sh; cndpfwd -c config.jsonc lb;"]
+      # args: ["-c", "./jsonc_gen.sh -k -p; cndpfwd -c config.jsonc lb;"] # to run in a kind cluster using bpf map pinning
       image: cndp
       imagePullPolicy: Never
       securityContext:
         capabilities:
           add:
             - NET_RAW
+            - IPC_LOCK
+            # - BPF # Enable if running in a kind cluster as the default kernel used is 5.15, BPF map support without privilege is only supported from 5.19.
       ports:
       - containerPort: 8094
         hostPort: 8094

diff --git a/doc/guides/sample_app_ug/cndpfwd.rst b/doc/guides/sample_app_ug/cndpfwd.rst
@@ -201,7 +201,7 @@ The configuration json file is located in the ``cndpfwd`` example sub-directory
         //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, tx-only-rx
         //                    acl-strict, acl-permissive
         //   uds_path   - (0) Path to unix domain socket to get xsk map fd
-        //   xsk_pin_path - (O) Path to pinned bpf map
+        //   xsk-pin-path - (O) Path to pinned bpf map
         "options": {
             "pkt_api": "xskdev",
             "no-metrics": false,

diff --git a/examples/cndpfwd/fwd.jsonc b/examples/cndpfwd/fwd.jsonc
@@ -124,7 +124,7 @@
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, tx-only-rx,
     //                    acl-strict, acl-permissive, [hyperscan | hs]
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
-    //   xsk_pin_path - (O) Path to pinned bpf map
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,

diff --git a/examples/cndpfwd/main.h b/examples/cndpfwd/main.h
@@ -61,7 +61,7 @@ enum {
 #define ENABLE_CLI_TAG       "cli"             /**< json tag to enable/disable CLI */
 #define MODE_TAG             "mode"            /**< json tag to set the mode flag */
 #define UDS_PATH_TAG         "uds_path"        /**< json tag for UDS to get xsk map fd */
-#define XSK_MAP_PIN_PATH_TAG "xsk_pin_path"    /**< json tag for pinned BPF map to get xsk map fd */
+#define XSK_MAP_PIN_PATH_TAG "xsk-pin-path"    /**< json tag for pinned BPF map to get xsk map fd */
 #define FIB_RULES_TAG        "l3fwd-fib-rules" /**< json tag to set up static FIB entries */
 #define HS_PATTERN_TAG       "hs-patterns"     /**< json tag for Hyperscan patterns */
 

diff --git a/examples/cnet-graph/cnet-graph.h b/examples/cnet-graph/cnet-graph.h
@@ -67,9 +67,10 @@ typedef enum {
     }
 // clang-format on
 
-#define NO_METRICS_TAG "no-metrics" /**< json tag for no-metrics */
-#define NO_RESTAPI_TAG "no-restapi" /**< json tag for no-restapi */
-#define ENABLE_CLI_TAG "cli"        /**< json tag to enable/disable CLI */
+#define NO_METRICS_TAG       "no-metrics"   /**< json tag for no-metrics */
+#define NO_RESTAPI_TAG       "no-restapi"   /**< json tag for no-restapi */
+#define ENABLE_CLI_TAG       "cli"          /**< json tag to enable/disable CLI */
+#define XSK_MAP_PIN_PATH_TAG "xsk-pin-path" /**< json tag for pinned BPF map to get xsk map fd */
 
 struct fwd_port {
     int lport;                           /**< PKTDEV lport id */
@@ -110,6 +111,7 @@ struct cnet_info {
     pthread_barrier_t barrier;                /**< Barrier for all threads */
     bool barrier_inited;                      /**< Barrier for all threads */
     graph_info_t graph_info[MAX_GRAPH_COUNT]; /**< Graph information */
+    char *xsk_map_path;                       /**< xsk_map pin path */
 };
 
 extern struct cnet_info *cinfo; /**< global application information pointer */

diff --git a/examples/cnet-graph/cnetfwd-graph.jsonc b/examples/cnet-graph/cnetfwd-graph.jsonc
@@ -124,6 +124,7 @@
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
+	//   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "no-metrics": false,
         "no-restapi": false,

diff --git a/examples/cnet-graph/parse-args.c b/examples/cnet-graph/parse-args.c
@@ -67,6 +67,10 @@ process_callback(jcfg_info_t *j, void *_obj, void *arg, int idx)
         } else if (!strncmp(obj.opt->name, ENABLE_CLI_TAG, nlen)) {
             if (obj.opt->val.type == BOOLEAN_OPT_TYPE)
                 ci->opts.cli = obj.opt->val.boolean;
+        } else if (!strncmp(obj.opt->name, XSK_MAP_PIN_PATH_TAG, nlen)) {
+            if (obj.opt->val.type == STRING_OPT_TYPE) {
+                ci->xsk_map_path = obj.opt->val.str;
+            }
         }
         break;
 

diff --git a/lib/core/xskdev/xskdev.c b/lib/core/xskdev/xskdev.c
@@ -756,7 +756,7 @@ xskdev_socket_create(struct lport_cfg *c)
             /* Try to retrieve the xsk_map_fd from a pinned bpf map*/
             xi->xsk_map_fd = bpf_obj_get(c->xsk_map_path);
             if (xi->xsk_map_fd < 0)
-                CNE_ERR_GOTO(err, "Failed to receive xsk map fd\n");
+                CNE_ERR_GOTO(err, "Failed to get xsk map fd from bpf object\n");
         }
 
         CNE_INFO("xi->xsk_map_fd = %d\n", xi->xsk_map_fd);

diff --git a/tools/jsonc_gen.sh b/tools/jsonc_gen.sh
@@ -37,7 +37,7 @@ if [ ${PINNED_BPF_MAP} = false ]  ; then
 UDS_OR_MAP_PATH="uds_path"
 UDS_OR_MAP="/tmp/afxdp.sock"
 else
-UDS_OR_MAP_PATH="xsk_pin_path"
+UDS_OR_MAP_PATH="xsk-pin-path"
 UDS_OR_MAP="/tmp/xsks_map"
 fi
 
@@ -83,25 +83,35 @@ function get_lcore()
     echo """${LCORE[index]}"""
 
 }
-output=$(lscpu | grep "NUMA node[0-9] CPU")
 
-# Parse the list of cores for each numa node and store it in an array
-IFS=$':\n';
-for each_output in $output
-do
-   if [ $((l%2)) -eq 0 ]
-   then
-      ((num_of_numa_nodes++))
-   else
-      # split the comma separated value of cores into an array
-      IFS=', ' read -ra list_of_numa_lcores <<< "$each_output"
+if [ $KIND = false ]  ; then
+    output=$(lscpu | grep "NUMA node[0-9] CPU")
+    # Parse the list of cores for each numa node and store it in an array
+    IFS=$':\n';
+    for each_output in $output
+    do
+    if [ $((l%2)) -eq 0 ]
+    then
+        ((num_of_numa_nodes++))
+    else
+        # split the comma separated value of cores into an array
+        IFS=', ' read -ra list_of_numa_lcores <<< "$each_output"
 
-      num_of_cores_in_each_numa_node[numa_node++]=${#list_of_numa_lcores[@]}
-      LCORE=("${LCORE[@]}" "${list_of_numa_lcores[@]}")
-   fi
-   ((l++))
-done
-unset IFS
+        num_of_cores_in_each_numa_node[numa_node++]=${#list_of_numa_lcores[@]}
+        LCORE=("${LCORE[@]}" "${list_of_numa_lcores[@]}")
+    fi
+    ((l++))
+    done
+    unset IFS
+else
+    output=$(lscpu -b -p=Core | grep -v '^#')
+    IFS=$'\n';
+    for each_output in $output
+    do
+        LCORE=("${LCORE[@]}" "$each_output")
+    done
+    unset IFS
+fi
 
 while [ $i -lt $num_of_interfaces ]
 do
@@ -140,9 +150,9 @@ EOF
     )
 if [ $KIND = false ]  ; then
     nic_numa_node=$(cat /sys/class/net/"${NET[i]}"/device/numa_node || echo 0)
-	lcore=$(get_lcore "$nic_numa_node" $i)
+    lcore=$(get_lcore "$nic_numa_node" $i)
 else
-    lcore=$(grep -c ^processor /proc/cpuinfo) # For kind you can't really retrieve the nic_numa_node as shown above
+    lcore=0 # For kind you can't really retrieve the nic_numa_node as shown above
 fi
     # create list of lcore groups
     lcore_groups[i]=$(
@@ -207,7 +217,7 @@ cat <<-EOF > ${config_file}
     //    description | desc - (O) Description of the umem space.
     "umems": {
         "umem0": {
-            "bufcnt": (16*$num_of_interfaces),
+            "bufcnt": $((num_of_interfaces * 16)),
             "bufsz": 2,
             "mtype": "2MB",
             "regions": [${regions[*]}
@@ -266,7 +276,7 @@ cat <<-EOF > ${config_file}
     //   cli        - (O) Enable/Disable CLI supported
     //   mode       - (O) Mode type [drop | rx-only], tx-only, [lb | loopback], fwd, acl-strict, acl-permissive
     //   uds_path   - (O) Path to unix domain socket to get xsk map fd
-    //   xsk_pin_path - (O) Path to pinned bpf map
+    //   xsk-pin-path - (O) Path to pinned bpf map
     "options": {
         "pkt_api": "xskdev",
         "no-metrics": false,