libdrgn: implement optimized x86-64 ELF relocations

After the libdwfl conversion, we apply ELF relocations with libdwfl
instead of our homegrown implementation. However, libdwfl is much slower
at it than the previous implementation. We can work around this by
(again) applying ELF relocations ourselves for architectures that we
care about (x86-64, to start). For other architectures, we can fall back
to libdwfl.

This new implementation of ELF relocation reworks the parallelization to
be per-file rather than per-relocation. The latter was done originally
because before commit 6f16ab0 ("libdrgn: only apply ELF relocations
to relocatable files"), we applied relocations to vmlinux, which is much
larger than most kernel modules. Now that we don't do that, it seems to
be slightly faster to parallelize by file.

libdrgn/Makefile.am

@@ -10,6 +10,8 @@ libdrgnimpl_la_SOURCES = binary_search_tree.h \
 			 dwarf_index.h \
 			 dwarf_info_cache.c \
 			 dwarf_info_cache.h \
+			 elf_relocator.c \
+			 elf_relocator.h \
 			 error.c \
 			 error.h \
 			 hash_table.c \

libdrgn/dwarf_index.c

@@ -16,6 +16,7 @@
 #include <sys/types.h>
 
 #include "internal.h"
+#include "elf_relocator.h"
 #include "dwarf_index.h"
 #include "read.h"
 #include "siphash.h"
@@ -432,6 +433,34 @@ static struct drgn_error *get_debug_sections(Elf *elf, Elf_Data **sections)
 	return NULL;
 }
 
+static struct drgn_error *apply_relocations(Dwfl_Module **modules,
+					    size_t num_modules)
+{
+	struct drgn_error *err;
+	struct drgn_elf_relocator relocator;
+	size_t i;
+
+	drgn_elf_relocator_init(&relocator);
+	for (i = 0; i < num_modules; i++) {
+		void **userdatap;
+		struct drgn_dwfl_module_userdata *userdata;
+
+		dwfl_module_info(modules[i], &userdatap, NULL, NULL, NULL, NULL,
+				 NULL, NULL);
+		userdata = *userdatap;
+		if (userdata->elf) {
+			err = drgn_elf_relocator_add_elf(&relocator,
+							 userdata->elf);
+			if (err)
+				goto out;
+		}
+	}
+	err = drgn_elf_relocator_apply(&relocator);
+out:
+	drgn_elf_relocator_deinit(&relocator);
+	return err;
+}
+
 static struct drgn_error *read_compilation_unit_header(const char *ptr,
 						       const char *end,
 						       struct compilation_unit *cu)
@@ -1521,6 +1550,9 @@ struct drgn_error *drgn_dwarf_index_update(struct drgn_dwarf_index *dindex,
 		err = &drgn_enomem;
 		goto out;
 	}
+	err = apply_relocations(modules.data, modules.size);
+	if (err)
+		goto out;
 	err = read_cus(dindex, modules.data, modules.size, &cus);
 	if (err)
 		goto out;

libdrgn/elf_relocator.c

@@ -0,0 +1,258 @@
+// Copyright 2018-2019 - Omar Sandoval
+// SPDX-License-Identifier: GPL-3.0+
+
+#include <inttypes.h>
+#include <string.h>
+
+#include "internal.h"
+#include "elf_relocator.h"
+
+DEFINE_VECTOR_FUNCTIONS(elf_vector)
+
+void drgn_elf_relocator_init(struct drgn_elf_relocator *relocator)
+{
+	elf_vector_init(&relocator->elfs);
+}
+
+void drgn_elf_relocator_deinit(struct drgn_elf_relocator *relocator)
+{
+	elf_vector_deinit(&relocator->elfs);
+}
+
+struct drgn_error *
+drgn_elf_relocator_add_elf(struct drgn_elf_relocator *relocator, Elf *elf)
+{
+	GElf_Ehdr ehdr_mem, *ehdr;
+
+	ehdr = gelf_getehdr(elf, &ehdr_mem);
+	if (!ehdr)
+		return drgn_error_libelf();
+
+	if (ehdr->e_type != ET_REL ||
+	    ehdr->e_machine != EM_X86_64 ||
+	    ehdr->e_ident[EI_CLASS] != ELFCLASS64 ||
+	    ehdr->e_ident[EI_DATA] !=
+	    (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ?
+	     ELFDATA2LSB : ELFDATA2MSB))
+		return NULL;
+
+	if (!elf_vector_append(&relocator->elfs, &elf))
+		return &drgn_enomem;
+	return NULL;
+}
+
+static struct drgn_error *apply_relocation(Elf_Data *data, uint64_t r_offset,
+					   uint32_t r_type, int64_t r_addend,
+					   uint64_t st_value)
+{
+	char *p;
+
+	p = (char *)data->d_buf + r_offset;
+	switch (r_type) {
+	case R_X86_64_NONE:
+		break;
+	case R_X86_64_32:
+		if (r_offset > SIZE_MAX - sizeof(uint32_t) ||
+		    r_offset + sizeof(uint32_t) > data->d_size) {
+			return drgn_error_create(DRGN_ERROR_ELF_FORMAT,
+						 "invalid relocation offset");
+		}
+		*(uint32_t *)p = st_value + r_addend;
+		break;
+	case R_X86_64_64:
+		if (r_offset > SIZE_MAX - sizeof(uint64_t) ||
+		    r_offset + sizeof(uint64_t) > data->d_size) {
+			return drgn_error_create(DRGN_ERROR_ELF_FORMAT,
+						 "invalid relocation offset");
+		}
+		*(uint64_t *)p = st_value + r_addend;
+		break;
+	default:
+		return drgn_error_format(DRGN_ERROR_ELF_FORMAT,
+					 "unimplemented relocation type %" PRIu32,
+					 r_type);
+	}
+	return NULL;
+}
+
+static struct drgn_error *relocate_section(Elf_Scn *scn, Elf_Scn *rela_scn,
+					   Elf_Scn *symtab_scn,
+					   uint64_t *sh_addrs, size_t shdrnum)
+{
+	struct drgn_error *err;
+	Elf_Data *data, *rela_data, *symtab_data;
+	const Elf64_Rela *relocs;
+	const Elf64_Sym *syms;
+	size_t num_relocs, num_syms;
+	size_t i;
+	GElf_Shdr *shdr, shdr_mem;
+
+	err = read_elf_section(scn, &data);
+	if (err)
+		return err;
+	err = read_elf_section(rela_scn, &rela_data);
+	if (err)
+		return err;
+	err = read_elf_section(symtab_scn, &symtab_data);
+	if (err)
+		return err;
+
+	relocs = (Elf64_Rela *)rela_data->d_buf;
+	num_relocs = rela_data->d_size / sizeof(Elf64_Rela);
+	syms = (Elf64_Sym *)symtab_data->d_buf;
+	num_syms = symtab_data->d_size / sizeof(Elf64_Sym);
+
+	for (i = 0; i < num_relocs; i++) {
+		const Elf64_Rela *reloc = &relocs[i];
+		uint32_t r_sym, r_type;
+		uint16_t st_shndx;
+		uint64_t sh_addr;
+
+		r_sym = ELF64_R_SYM(reloc->r_info);
+		r_type = ELF64_R_TYPE(reloc->r_info);
+
+		if (r_sym >= num_syms) {
+			return drgn_error_create(DRGN_ERROR_ELF_FORMAT,
+						 "invalid relocation symbol");
+		}
+		st_shndx = syms[r_sym].st_shndx;
+		if (st_shndx == 0) {
+			sh_addr = 0;
+		} else if (st_shndx < shdrnum) {
+			sh_addr = sh_addrs[st_shndx - 1];
+		} else {
+			return drgn_error_create(DRGN_ERROR_ELF_FORMAT,
+						 "invalid symbol section index");
+		}
+		err = apply_relocation(data, reloc->r_offset, r_type,
+				       reloc->r_addend,
+				       sh_addr + syms[r_sym].st_value);
+		if (err)
+			return err;
+	}
+
+	/*
+	 * Mark the relocation section as empty so that libdwfl doesn't try to
+	 * apply it again.
+	 */
+	shdr = gelf_getshdr(rela_scn, &shdr_mem);
+	if (!shdr)
+		return drgn_error_libelf();
+	shdr->sh_size = 0;
+	if (!gelf_update_shdr(rela_scn, shdr))
+		return drgn_error_libelf();
+	rela_data->d_size = 0;
+	return NULL;
+}
+
+static struct drgn_error *relocate_elf(Elf *elf)
+{
+	struct drgn_error *err;
+	size_t shdrnum, shstrndx;
+	uint64_t *sh_addrs;
+	Elf_Scn *scn;
+
+	if (elf_getshdrnum(elf, &shdrnum))
+		return drgn_error_libelf();
+	if (shdrnum > 1) {
+		sh_addrs = calloc(shdrnum - 1, sizeof(*sh_addrs));
+		if (!sh_addrs)
+			return &drgn_enomem;
+
+		scn = NULL;
+		while ((scn = elf_nextscn(elf, scn))) {
+			size_t ndx;
+
+			ndx = elf_ndxscn(scn);
+			if (ndx > 0 && ndx < shdrnum) {
+				GElf_Shdr *shdr, shdr_mem;
+
+				shdr = gelf_getshdr(scn, &shdr_mem);
+				if (!shdr) {
+					err = drgn_error_libelf();
+					goto out;
+				}
+				sh_addrs[ndx - 1] = shdr->sh_addr;
+			}
+		}
+	} else {
+		sh_addrs = NULL;
+	}
+
+	if (elf_getshdrstrndx(elf, &shstrndx)) {
+		err = drgn_error_libelf();
+		goto out;
+	}
+
+	scn = NULL;
+	while ((scn = elf_nextscn(elf, scn))) {
+		GElf_Shdr *shdr, shdr_mem;
+		const char *scnname;
+
+		shdr = gelf_getshdr(scn, &shdr_mem);
+		if (!shdr) {
+			err = drgn_error_libelf();
+			goto out;
+		}
+
+		if (shdr->sh_type != SHT_RELA)
+			continue;
+
+		scnname = elf_strptr(elf, shstrndx, shdr->sh_name);
+		if (!scnname)
+			continue;
+
+		if (strncmp(scnname, ".rela.debug_", 12) == 0) {
+			Elf_Scn *info_scn, *link_scn;
+
+			info_scn = elf_getscn(elf, shdr->sh_info);
+			if (!info_scn) {
+				err = drgn_error_libelf();
+				goto out;
+			}
+
+			link_scn = elf_getscn(elf, shdr->sh_link);
+			if (!link_scn) {
+				err = drgn_error_libelf();
+				goto out;
+			}
+
+			err = relocate_section(info_scn, scn, link_scn,
+					       sh_addrs, shdrnum);
+			if (err)
+				goto out;
+		}
+	}
+out:
+	free(sh_addrs);
+	return NULL;
+}
+
+struct drgn_error *
+drgn_elf_relocator_apply(struct drgn_elf_relocator *relocator)
+{
+	struct drgn_error *err = NULL;
+	Elf **elfs = relocator->elfs.data;
+	size_t num_elfs = relocator->elfs.size;
+
+	#pragma omp parallel for schedule(dynamic)
+	for (size_t i = 0; i < num_elfs; i++) {
+		struct drgn_error *err2;
+
+		if (err)
+			continue;
+
+		err2 = relocate_elf(elfs[i]);
+		if (err2) {
+			#pragma omp critical(relocators_err)
+			{
+				if (err)
+					drgn_error_destroy(err2);
+				else
+					err = err2;
+			}
+			continue;
+		}
+	}
+	return err;
+}

libdrgn/elf_relocator.h

@@ -0,0 +1,71 @@
+// Copyright 2019 - Omar Sandoval
+// SPDX-License-Identifier: GPL-3.0+
+
+/**
+ * @file
+ *
+ * ELF relocator.
+ *
+ * See @ref ElfRelocator.
+ */
+
+#ifndef DRGN_ELF_RELOCATOR_H
+#define DRGN_ELF_RELOCATOR_H
+
+#include <libelf.h>
+
+#include "vector.h"
+
+/**
+ * @ingroup Internals.
+ *
+ * @defgroup ElfRelocator ELF relocator
+ *
+ * Fast ELF relocations.
+ *
+ * Before the debugging information in a relocatable ELF file (e.g., Linux
+ * kernel module) can be used, it must have ELF relocations applied. This is
+ * usually done by libdwfl. However, libdwfl is relatively slow at it. @ref
+ * drgn_elf_relocator is a much faster, parallelized implementation of ELF
+ * relocation. It is only implemented for x86-64; for other architectures, we
+ * can fall back to libdwfl.
+ */
+
+DEFINE_VECTOR_TYPE(elf_vector, Elf *)
+
+/**
+ * ELF relocation interface.
+ *
+ * This interface is used to apply ELF relocations to debug sections in ELF
+ * files.
+ *
+ * A relocator is initialized with @ref drgn_elf_relocator_init(). Files to be
+ * relocated are added with @ref drgn_elf_relocator_add_elf(). Once all files
+ * have been added, relocations are applied with @ref
+ * drgn_elf_relocator_apply(). Finally, the relocator must be cleaned up with
+ * @ref drgn_elf_relocator_deinit().
+ */
+struct drgn_elf_relocator {
+	struct elf_vector elfs;
+};
+
+/** Initialize a @ref drgn_elf_relocator. */
+void drgn_elf_relocator_init(struct drgn_elf_relocator *relocator);
+
+/** Deinitialize a @ref drgn_elf_relocator. */
+void drgn_elf_relocator_deinit(struct drgn_elf_relocator *relocator);
+
+/**
+ * Add an ELF file to be relocated by a @ref drgn_elf_relocator.
+ *
+ * If the ELF file is not relocatable or has an unsupported architecture, this
+ * does nothing.
+ */
+struct drgn_error *
+drgn_elf_relocator_add_elf(struct drgn_elf_relocator *relocator, Elf *elf);
+
+/** Apply ELF relocations to all files added to a @ref drgn_elf_relocator. */
+struct drgn_error *
+drgn_elf_relocator_apply(struct drgn_elf_relocator *relocator);
+
+#endif /* DRGN_ELF_RELOCATOR_H */