Add support for bpf2bpf function calls

Signed-off-by: Dave Thaler <[email protected]>

src/asm_cfg.cpp

@@ -35,11 +35,99 @@ static bool has_fall(Instruction ins) {
     return true;
 }
 
+/// Update a control-flow graph to inline function macros.
+static void add_cfg_nodes(cfg_t& cfg, label_t caller_label, label_t entry_label) {
+    bool first = true;
+
+    // Get the label of the node to go to on returning from the macro.
+    basic_block_t& exit_to_node = cfg.get_node(cfg.next_nodes(caller_label).front());
+
+    // Construct the variable prefix to use for the new stack frame,
+    // and store a copy in the CallLocal instruction since the instruction-specific
+    // labels may only exist until the CFG is simplified.
+    basic_block_t& caller_node = cfg.get_node(caller_label);
+    std::string stack_frame_prefix = to_string(caller_label);
+    for (auto& inst : caller_node) {
+        if (std::holds_alternative<CallLocal>(inst)) {
+            std::get<CallLocal>(inst).stack_frame_prefix = stack_frame_prefix;
+        }
+    }
+
+    // Walk the transitive closure of CFG nodes starting at entry_label and ending at
+    // any exit instruction,
+    std::list<label_t> macro_labels;
+    macro_labels.push_back(entry_label);
+    for (auto it = macro_labels.begin(); it != macro_labels.end(); it++) {
+        label_t macro_label = *it;
+
+        // Clone the macro block into a new block with the new stack frame prefix.
+        const label_t label(macro_label.from, macro_label.to, stack_frame_prefix);
+        auto& bb = cfg.insert(label);
+        for (auto inst : cfg.get_node(macro_label)) {
+            if (std::holds_alternative<Exit>(inst)) {
+                std::get<Exit>(inst).stack_frame_prefix = label.stack_frame_prefix;
+            } else if (std::holds_alternative<Call>(inst)) {
+                std::get<Call>(inst).stack_frame_prefix = label.stack_frame_prefix;
+            }
+            bb.insert(inst);
+        }
+
+        if (first) {
+            // Add an edge from the caller to the new block.
+            first = false;
+            caller_node >> bb;
+        }
+
+        // Add an edge from any other predecessors.
+        const auto& prev_macro_nodes = cfg.prev_nodes(macro_label);
+        for (const auto& prev_macro_label : prev_macro_nodes) {
+            const label_t prev_label(prev_macro_label.from, prev_macro_label.to, to_string(caller_label));
+            const auto& labels = cfg.labels();
+            if (std::find(labels.begin(), labels.end(), prev_label) != labels.end())
+                cfg.get_node(prev_label) >> bb;
+        }
+
+        // Walk all successor nodes.
+        const auto& next_macro_nodes = cfg.next_nodes(macro_label);
+        for (const auto& next_macro_label : next_macro_nodes) {
+            if (next_macro_label == cfg.exit_label()) {
+                // This is an exit transition, so add edge to the block to execute
+                // upon returning from the macro.
+                bb >> exit_to_node;
+            } else if (std::find(macro_labels.begin(), macro_labels.end(), next_macro_label) == macro_labels.end()) {
+                // Push any other unprocessed successor label onto the list to be processed.
+                macro_labels.push_back(next_macro_label);
+            }
+        }
+    }
+
+    // Remove the original edge from the caller node to its successor,
+    // since processing now goes through the function macro instead.
+    caller_node -= exit_to_node;
+
+    // Finally, recurse to replace any nested function macros.
+    string caller_label_str = to_string(caller_label);
+    int stack_frame_depth = std::count(caller_label_str.begin(), caller_label_str.end(), STACK_FRAME_DELIMITER) + 2;
+    const int MAX_CALL_STACK_FRAMES = 8;
+    for (auto& macro_label : macro_labels) {
+        const label_t label(macro_label.from, macro_label.to, caller_label_str);
+        for (const auto inst : cfg.get_node(label)) {
+            if (std::holds_alternative<CallLocal>(inst)) {
+                if (stack_frame_depth >= MAX_CALL_STACK_FRAMES)
+                    throw std::runtime_error{"too many call stack frames"};
+                add_cfg_nodes(cfg, label, std::get<CallLocal>(inst).target);
+            }
+        }
+    }
+}
+
 /// Convert an instruction sequence to a control-flow graph (CFG).
 static cfg_t instruction_seq_to_cfg(const InstructionSeq& insts, bool must_have_exit) {
     cfg_t cfg;
     std::optional<label_t> falling_from = {};
     bool first = true;
+
+    // Do a first pass ignoring all function macro calls.
     for (const auto& [label, inst, _] : insts) {
 
         if (std::holds_alternative<Undefined>(inst))
@@ -73,6 +161,15 @@ static cfg_t instruction_seq_to_cfg(const InstructionSeq& insts, bool must_have_
             cfg.get_node(*falling_from) >> cfg.get_node(cfg.exit_label());
     }
 
+    // Now replace macros. We have to do this as a second pass so that
+    // we only add new nodes that are actually reachable, based on the
+    // results of the first pass.
+    for (auto& [label, inst, _] : insts) {
+        if (std::holds_alternative<CallLocal>(inst)) {
+            add_cfg_nodes(cfg, label, std::get<CallLocal>(inst).target);
+        }
+    }
+
     return cfg;
 }
 

src/asm_files.cpp

@@ -71,7 +71,7 @@ std::tuple<string, ELFIO::Elf_Half> get_symbol_name_and_section_index(ELFIO::con
     return {symbol_name, section_index};
 }
 
-std::tuple<ELFIO::Elf64_Addr, unsigned char> get_value(ELFIO::const_symbol_section_accessor& symbols, ELFIO::Elf_Word index) {
+std::tuple<ELFIO::Elf64_Addr, unsigned char> get_value(const ELFIO::const_symbol_section_accessor& symbols, ELFIO::Elf_Word index) {
     string symbol_name;
     ELFIO::Elf64_Addr value{};
     ELFIO::Elf_Xword size{};
@@ -152,6 +152,45 @@ std::tuple<string, ELFIO::Elf_Xword> get_program_name_and_size(ELFIO::section& s
     return {program_name, size};
 }
 
+void relocate_function(ebpf_inst& inst, ELFIO::Elf64_Addr offset, ELFIO::Elf_Word index, const ELFIO::const_symbol_section_accessor& symbols) {
+    auto [relocation_offset, relocation_type] = get_value(symbols, index);
+    inst.imm = ((relocation_offset - offset) / sizeof(ebpf_inst)) - 1;
+}
+
+void relocate_map(ebpf_inst& inst, const std::string& symbol_name, const std::variant<size_t, std::map<std::string, size_t>>& map_record_size_or_map_offsets, const program_info& info, ELFIO::Elf64_Addr offset, ELFIO::Elf_Word index, const ELFIO::const_symbol_section_accessor& symbols) {
+    // Only permit loading the address of the map.
+    if ((inst.opcode & INST_CLS_MASK) != INST_CLS_LD) {
+        throw std::runtime_error("Illegal operation on symbol " + symbol_name + " at location " +
+                                 std::to_string(offset / sizeof(ebpf_inst)));
+    }
+    inst.src = 1; // magic number for LoadFd
+
+    // Relocation value is an offset into the "maps" or ".maps" section.
+    auto [relocation_offset, relocation_type] = get_value(symbols, index);
+    if (map_record_size_or_map_offsets.index() == 0) {
+        // The older maps section format uses a single map_record_size value, so we can
+        // calculate the map descriptor index directly.
+        size_t reloc_value = relocation_offset / std::get<0>(map_record_size_or_map_offsets);
+        if (reloc_value >= info.map_descriptors.size()) {
+            throw std::runtime_error("Bad reloc value (" + std::to_string(reloc_value) + "). " +
+                                     "Make sure to compile with -O2.");
+        }
+
+        inst.imm = info.map_descriptors.at(reloc_value).original_fd;
+    } else {
+        // The newer .maps section format uses a variable-length map descriptor array,
+        // so we need to look up the map descriptor index in a map.
+        auto& map_descriptors_offsets = std::get<1>(map_record_size_or_map_offsets);
+        auto it = map_descriptors_offsets.find(symbol_name);
+
+        if (it == map_descriptors_offsets.end()) {
+            throw std::runtime_error("Bad reloc value (" + std::to_string(index) + "). " +
+                                     "Make sure to compile with -O2.");
+        }
+        inst.imm = info.map_descriptors.at(it->second).original_fd;
+    }
+}
+
 vector<raw_program> read_elf(std::istream& input_stream, const std::string& path, const std::string& desired_section, const ebpf_verifier_options_t* options, const ebpf_platform_t* platform) {
     if (options == nullptr)
         options = &ebpf_verifier_default_options;
@@ -292,46 +331,22 @@ vector<raw_program> read_elf(std::istream& input_stream, const std::string& path
 
                     auto [symbol_name, symbol_section_index] = get_symbol_name_and_section_index(symbols, index);
 
-                    // Only perform relocation for symbols located in the maps section.
-                    if (!map_section_indices.contains(symbol_section_index)) {
-                        std::string unresolved_symbol = "Unresolved external symbol " + symbol_name +
-                                                        " in section " + name + " at location " + std::to_string(offset / sizeof(ebpf_inst));
-                        unresolved_symbols.push_back(unresolved_symbol);
+                    // Perform relocation for function symbols.
+                    if ((inst.opcode == INST_OP_CALL) && (inst.src == INST_CALL_LOCAL) &&
+                        (reader.sections[symbol_section_index] == section.get())) {
+                        relocate_function(inst, offset, index, symbols);
                         continue;
                     }
 
-                    // Only permit loading the address of the map.
-                    if ((inst.opcode & INST_CLS_MASK) != INST_CLS_LD) {
-                        throw std::runtime_error("Illegal operation on symbol " + symbol_name +
-                                                 " at location " + std::to_string(offset / sizeof(ebpf_inst)));
-                    }
-                    inst.src = 1; // magic number for LoadFd
-
-                    // Relocation value is an offset into the "maps" or ".maps" section.
-                    auto [relocation_offset, relocation_type] = get_value(symbols, index);
-                    if (map_record_size_or_map_offsets.index() == 0) {
-                        // The older maps section format uses a single map_record_size value, so we can
-                        // calculate the map descriptor index directly.
-                        size_t reloc_value = relocation_offset / std::get<0>(map_record_size_or_map_offsets);
-                        if (reloc_value >= info.map_descriptors.size()) {
-                            throw std::runtime_error("Bad reloc value (" + std::to_string(reloc_value) + "). "
-                                                    + "Make sure to compile with -O2.");
-                        }
-
-                        inst.imm = info.map_descriptors.at(reloc_value).original_fd;
-                    }
-                    else {
-                        // The newer .maps section format uses a variable-length map descriptor array,
-                        // so we need to look up the map descriptor index in a map.
-                        auto& map_descriptors_offsets = std::get<1>(map_record_size_or_map_offsets);
-                        auto it = map_descriptors_offsets.find(symbol_name);
-
-                        if (it == map_descriptors_offsets.end()) {
-                            throw std::runtime_error("Bad reloc value (" + std::to_string(index) + "). "
-                                                    + "Make sure to compile with -O2.");
-                        }
-                        inst.imm = info.map_descriptors.at(it->second).original_fd;
+                    // Perform relocation for symbols located in the maps section.
+                    if (map_section_indices.contains(symbol_section_index)) {
+                        relocate_map(inst, symbol_name, map_record_size_or_map_offsets, info, offset, index, symbols);
+                        continue;
                     }
+
+                    std::string unresolved_symbol = "Unresolved external symbol " + symbol_name +
+                                                    " in section " + name + " at location " + std::to_string(offset / sizeof(ebpf_inst));
+                    unresolved_symbols.push_back(unresolved_symbol);
                 }
             }
             prog.line_info.resize(prog.prog.size());

src/asm_marshal.cpp

@@ -175,13 +175,21 @@ struct MarshalVisitor {
 
     vector<ebpf_inst> operator()(Call const& b) {
         return {
-            ebpf_inst{.opcode = static_cast<uint8_t>(INST_OP_CALL | INST_SRC_IMM), .dst = 0, .src = 0, .offset = 0, .imm = b.func}};
+            ebpf_inst{.opcode = static_cast<uint8_t>(INST_OP_CALL), .dst = 0, .src = INST_CALL_STATIC_HELPER, .offset = 0, .imm = b.func}};
+    }
+
+    vector<ebpf_inst> operator()(CallLocal const& b) {
+        return {ebpf_inst{.opcode = static_cast<uint8_t>(INST_OP_CALL),
+                          .dst = 0,
+                          .src = INST_CALL_LOCAL,
+                          .offset = 0,
+                          .imm = label_to_offset32(b.target)}};
     }
 
     vector<ebpf_inst> operator()(Callx const& b) {
         // callx is defined to have the register in 'dst' not in 'src'.
         return {
-            ebpf_inst{.opcode = static_cast<uint8_t>(INST_OP_CALL | INST_SRC_REG), .dst = b.func.v, .src = 0, .offset = 0}};
+            ebpf_inst{.opcode = static_cast<uint8_t>(INST_OP_CALLX), .dst = b.func.v, .src = INST_CALL_STATIC_HELPER, .offset = 0}};
     }
 
     vector<ebpf_inst> operator()(Exit const& b) {

src/asm_ostream.cpp

@@ -11,6 +11,9 @@
 #include "crab/cfg.hpp"
 #include "crab/interval.hpp"
 #include "crab/variable.hpp"
+#include "helpers.hpp"
+#include "platform.hpp"
+#include "spec_type_descriptors.hpp"
 
 using std::optional;
 using std::string;
@@ -160,6 +163,11 @@ std::ostream& operator<<(std::ostream& os, ValidSize const& a) {
     return os << a.reg << ".value" << op << 0;
 }
 
+std::ostream& operator<<(std::ostream& os, ValidCall const& a) {
+    EbpfHelperPrototype proto = global_program_info->platform->get_helper_prototype(a.func);
+    return os << "valid call(" << proto.name << ")";
+}
+
 std::ostream& operator<<(std::ostream& os, ValidMapKeyValue const& a) {
     return os << "within stack(" << a.access_reg << ":" << (a.key ? "key_size" : "value_size") << "(" << a.map_fd_reg << "))";
 }
@@ -266,6 +274,8 @@ struct InstructionPrinterVisitor {
         os_ << ")";
     }
 
+    void operator()(CallLocal const& call) { os_ << "call <" << to_string(call.target) << ">"; }
+
     void operator()(Callx const& callx) { os_ << "callx " << callx.func; }
 
     void operator()(Exit const& b) { os_ << "exit"; }
@@ -531,6 +541,10 @@ std::ostream& operator<<(std::ostream& o, const crab::basic_block_rev_t& bb) {
 std::ostream& operator<<(std::ostream& o, const cfg_t& cfg) {
     for (const label_t& label : cfg.sorted_labels()) {
         o << cfg.get_node(label);
+        o << "edges to:";
+        for (const label_t& next_label : cfg.next_nodes(label))
+            o << " " << next_label;
+        o << "\n";
     }
     return o;
 }

src/asm_parse.cpp

@@ -157,6 +157,9 @@ Instruction parse_instruction(const std::string& line, const std::map<std::strin
         int func = boost::lexical_cast<int>(m[1]);
         return make_call(func, g_ebpf_platform_linux);
     }
+    if (regex_match(text, m, regex("call " WRAPPED_LABEL))) {
+        return CallLocal{.target = label_name_to_label.at(m[1])};
+    }
     if (regex_match(text, m, regex("callx " REG))) {
         return Callx{reg(m[1])};
     }