Add support for reading symbols from /tmp/perf-pid.map

This adds basic support for /tmp/perf-pid.map. To cope with processes in containers, it supports: * mapping from BCC's PID namespace to the target process's PID namespace using /proc/pid/status * resolving a target process's root filesystem using /proc/pid/root
iovisor · 4ast · Jun 20, 2016 · Jun 15, 2016 · Jun 15, 2016 · 769edf959f8dc5cb67a08246d461885eb92ed687
commit 769edf959f8dc5cb67a08246d461885eb92ed687
@@ -33,11 +33,11 @@ if (CMAKE_COMPILER_IS_GNUCC)
   endif()
 endif()
 
-add_library(bcc-shared SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c shared_table.cc exported_files.cc bcc_elf.c bcc_proc.c bcc_syms.cc usdt_args.cc usdt.cc)
+add_library(bcc-shared SHARED bpf_common.cc bpf_module.cc libbpf.c perf_reader.c shared_table.cc exported_files.cc bcc_elf.c bcc_perf_map.c bcc_proc.c bcc_syms.cc usdt_args.cc usdt.cc)
 set_target_properties(bcc-shared PROPERTIES VERSION ${REVISION_LAST} SOVERSION 0)
 set_target_properties(bcc-shared PROPERTIES OUTPUT_NAME bcc)
 
-add_library(bcc-loader-static libbpf.c perf_reader.c bcc_elf.c bcc_proc.c)
+add_library(bcc-loader-static libbpf.c perf_reader.c bcc_elf.c bcc_perf_map.c bcc_proc.c)
 add_library(bcc-static STATIC bpf_common.cc bpf_module.cc shared_table.cc exported_files.cc bcc_syms.cc usdt_args.cc usdt.cc)
 set_target_properties(bcc-static PROPERTIES OUTPUT_NAME bcc)
 

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "bcc_perf_map.h"
+
+int bcc_perf_map_nspid(int pid) {
+  char status_path[64];
+  FILE *status;
+
+  snprintf(status_path, sizeof(status_path), "/proc/%d/status", pid);
+  status = fopen(status_path, "r");
+
+  if (!status)
+    return -1;
+
+  // return the original PID if the NSpid line is missing
+  int nspid = pid;
+
+  size_t size;
+  char *line = NULL;
+  while (getline(&line, &size, status) != -1)
+    if (strstr(line, "NSpid:") != NULL)
+      // PID namespaces can be nested -- last number is innermost PID
+      nspid = (int)strtol(strrchr(line, '\t'), NULL, 10);
+  free(line);
+
+  return nspid;
+}
+
+bool bcc_perf_map_path(char *map_path, size_t map_len, int pid) {
+  char source[64];
+  snprintf(source, sizeof(source), "/proc/%d/root", pid);
+
+  char target[4096];
+  ssize_t target_len = readlink(source, target, sizeof(target) - 1);
+  if (target_len == -1)
+    return false;
+
+  target[target_len] = '\0';
+  if (strcmp(target, "/") == 0)
+    target[0] = '\0';
+
+  int nspid = bcc_perf_map_nspid(pid);
+
+  snprintf(map_path, map_len, "%s/tmp/perf-%d.map", target, nspid);
+  return true;
+}
+
+int bcc_perf_map_foreach_sym(const char *path, bcc_perf_map_symcb callback,
+                             void* payload) {
+  FILE* file = fopen(path, "r");
+  if (!file)
+    return -1;
+
+  char *line = NULL;
+  size_t size;
+  long long begin, len;
+  while (getline(&line, &size, file) != -1) {
+    char *cursor = line;
+    char *newline, *sep;
+
+    begin = strtoull(cursor, &sep, 16);
+    if (*sep != ' ' || (sep == cursor && begin == 0))
+      continue;
+    cursor = sep;
+    while (*cursor && isspace(*cursor)) cursor++;
+
+    len = strtoull(cursor, &sep, 16);
+    if (*sep != ' ' || (sep == cursor && begin == 0))
+      continue;
+    cursor = sep;
+    while (*cursor && isspace(*cursor)) cursor++;
+
+    newline = strchr(cursor, '\n');
+    if (newline)
+        newline[0] = '\0';
+
+    callback(cursor, begin, len - 1, 0, payload);
+  }
+
+  free(line);
+  fclose(file);
+
+  return 0;
+}
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2016 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBBCC_PERF_MAP_H
+#define LIBBCC_PERF_MAP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <unistd.h>
+
+typedef int (*bcc_perf_map_symcb)(const char *, uint64_t, uint64_t, int,
+                                  void *);
+
+int bcc_perf_map_nspid(int pid);
+bool bcc_perf_map_path(char *map_path, size_t map_len, int pid);
+int bcc_perf_map_foreach_sym(const char *path, bcc_perf_map_symcb callback,
+                             void* payload);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
@@ -25,6 +25,7 @@
 #include <ctype.h>
 #include <stdio.h>
 
+#include "bcc_perf_map.h"
 #include "bcc_proc.h"
 #include "bcc_elf.h"
 
@@ -85,7 +86,7 @@ int bcc_procutils_each_module(int pid, bcc_procutils_modulecb callback,
     char perm[8], dev[8];
     long long begin, end, size, inode;
 
-    ret = fscanf(procmap, "%llx-%llx %s %llx %s %llx", &begin, &end, perm,
+    ret = fscanf(procmap, "%llx-%llx %s %llx %s %lld", &begin, &end, perm,
                  &size, dev, &inode);
 
     if (!fgets(endline, sizeof(endline), procmap))
@@ -108,6 +109,13 @@ int bcc_procutils_each_module(int pid, bcc_procutils_modulecb callback,
   } while (ret && ret != EOF);
 
   fclose(procmap);
+
+  // Add a mapping to /tmp/perf-pid.map for the entire address space. This will
+  // be used if symbols aren't resolved in an earlier mapping.
+  char map_path[4096];
+  if (bcc_perf_map_path(map_path, sizeof(map_path), pid))
+    callback(map_path, 0, -1, payload);
+
   return 0;
 }
 

@@ -20,6 +20,7 @@
 #include <unistd.h>
 
 #include "bcc_elf.h"
+#include "bcc_perf_map.h"
 #include "bcc_proc.h"
 #include "bcc_syms.h"
 
@@ -139,11 +140,18 @@ bool ProcSyms::Module::is_so() const {
   return strstr(name_.c_str(), ".so") != nullptr;
 }
 
+bool ProcSyms::Module::is_perf_map() const {
+  return strstr(name_.c_str(), ".map") != nullptr;
+}
+
 void ProcSyms::Module::load_sym_table() {
   if (syms_.size())
     return;
 
-  bcc_elf_foreach_sym(name_.c_str(), _add_symbol, this);
+  if (is_perf_map())
+    bcc_perf_map_foreach_sym(name_.c_str(), _add_symbol, this);
+  else
+    bcc_elf_foreach_sym(name_.c_str(), _add_symbol, this);
 }
 
 bool ProcSyms::Module::find_name(const char *symname, uint64_t *addr) {

@@ -83,6 +83,7 @@ class ProcSyms : SymbolCache {
     bool find_addr(uint64_t addr, struct bcc_symbol *sym);
     bool find_name(const char *symname, uint64_t *addr);
     bool is_so() const;
+    bool is_perf_map() const;
 
     static int _add_symbol(const char *symname, uint64_t start, uint64_t end,
                            int flags, void *p);

@@ -16,11 +16,14 @@
 #include <dlfcn.h>
 #include <stdint.h>
 #include <string.h>
+#include <sys/mman.h>
 #include <unistd.h>
 
 #include "bcc_elf.h"
+#include "bcc_perf_map.h"
 #include "bcc_proc.h"
 #include "bcc_syms.h"
+#include "vendor/tinyformat.hpp"
 
 #include "catch.hpp"
 
@@ -109,3 +112,82 @@ TEST_CASE("resolve symbol addresses for a given PID", "[c_api]") {
     REQUIRE(string("strtok") == sym.name);
   }
 }
+
+#define STACK_SIZE (1024 * 1024)
+static char child_stack[STACK_SIZE];
+
+static string perf_map_path(pid_t pid) {
+  return tfm::format("/tmp/perf-%d.map", pid);
+}
+
+static int child_func(void *arg) {
+  unsigned long long map_addr = (unsigned long long)arg;
+
+  const char *path = perf_map_path(getpid()).c_str();
+  FILE *file = fopen(path, "w");
+  if (file == NULL) {
+    return -1;
+  }
+  fprintf(file, "%llx 10 dummy_fn\n", map_addr);
+  fclose(file);
+
+  sleep(5);
+
+  unlink(path);
+  return 0;
+}
+
+static pid_t spawn_child(void *map_addr, bool own_pidns) {
+  int flags = 0;
+  if (own_pidns)
+    flags |= CLONE_NEWPID;
+
+  pid_t child = clone(child_func, /* stack grows down */ child_stack + STACK_SIZE,
+      flags, (void*)map_addr);
+  if (child < 0)
+    return -1;
+
+  sleep(1); // let the child get set up
+  return child;
+}
+
+TEST_CASE("resolve symbols using /tmp/perf-pid.map", "[c_api]") {
+  const int map_sz = 4096;
+  void *map_addr = mmap(NULL, map_sz, PROT_READ | PROT_EXEC,
+    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  REQUIRE(map_addr != MAP_FAILED);
+
+  struct bcc_symbol sym;
+  pid_t child = -1;
+
+  SECTION("same namespace") {
+    child = spawn_child(map_addr, /* own_pidns */ false);
+    REQUIRE(child > 0);
+
+    void *resolver = bcc_symcache_new(child);
+    REQUIRE(resolver);
+
+    REQUIRE(bcc_symcache_resolve(resolver, (unsigned long long)map_addr,
+        &sym) == 0);
+    REQUIRE(sym.module);
+    REQUIRE(string(sym.module) == perf_map_path(child));
+    REQUIRE(string("dummy_fn") == sym.name);
+  }
+
+  SECTION("separate namespace") {
+    child = spawn_child(map_addr, /* own_pidns */ true);
+    REQUIRE(child > 0);
+
+    void *resolver = bcc_symcache_new(child);
+    REQUIRE(resolver);
+
+    REQUIRE(bcc_symcache_resolve(resolver, (unsigned long long)map_addr,
+        &sym) == 0);
+    REQUIRE(sym.module);
+    // child is PID 1 in its namespace
+    REQUIRE(string(sym.module) == perf_map_path(1));
+    REQUIRE(string("dummy_fn") == sym.name);
+  }
+
+  munmap(map_addr, map_sz);
+}