Cross compile for armhf on Cortex-A5 results in illegal instruction

[aka-mj]

Zig Version

0.11.0-dev.4271+1aacfa718

Steps to Reproduce and Observed Behavior

I have the following build.zig, which specifies the target as arm-linux-gnueabi, as this is built for an embedded device.

const std = @import("std");

pub fn build(b: *std.Build) void {
    const target = b.standardTargetOptions(.{});

    const optimize = b.standardOptimizeOption(.{});

    const exe = b.addExecutable(.{
        .name = "wifiscan",
        .root_source_file = .{ .path = "src/main.zig" },
        .target = .{ .cpu_arch = std.Target.Cpu.Arch.arm, .os_tag = std.Target.Os.Tag.linux, .abi = std.Target.Abi.gnueabi },
        .optimize = optimize,
    });
    exe.addCSourceFile("src/wifi_scan.c", &[_][]const u8{"-std=c99"});
    exe.linkSystemLibrary("c");
    exe.addIncludePath("src");
    exe.addIncludePath("libs/libmnl/include");
    exe.addObjectFile("libs/libmnl/zig-out/lib/libmnl.a");

    b.installArtifact(exe);

    const run_cmd = b.addRunArtifact(exe);

    run_cmd.step.dependOn(b.getInstallStep());

    if (b.args) |args| {
        run_cmd.addArgs(args);
    }

    const run_step = b.step("run", "Run the app");
    run_step.dependOn(&run_cmd.step);

    const unit_tests = b.addTest(.{
        .root_source_file = .{ .path = "src/main.zig" },
        .target = target,
        .optimize = optimize,
    });

    const run_unit_tests = b.addRunArtifact(unit_tests);

    const test_step = b.step("test", "Run unit tests");
    test_step.dependOn(&run_unit_tests.step);
}

I build via zig build.

When examining the file on the host it looks like its built for the target.

> file zig-out/bin/wifiscan 
zig-out/bin/wifiscan: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux.so.3, for GNU/Linux 2.0.0, with debug_info, not stripped

But running on the target gives an error.

~> strace /bin/wifiscan
execve("/bin/wifiscan", ["/bin/wifiscan"], 0xbea04da0 /* 13 vars */) = -1 ENOENT (No such file or directory)
/bin/strace: exec: No such file or directory
+++ exited with 1 +++

Looking at the file further on the host it appears the interpreter is from the host rather then the target.

> readelf -l zig-out/bin/wifiscan 

Elf file type is EXEC (Executable file)
Entry point 0x2d088
There are 12 program headers, starting at offset 52

Program Headers:
  Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
  PHDR           0x000034 0x00010034 0x00010034 0x00180 0x00180 R   0x4
  INTERP         0x0001b4 0x000101b4 0x000101b4 0x00013 0x00013 R   0x1
      [Requesting program interpreter: /lib/ld-linux.so.3]
  LOAD           0x000000 0x00010000 0x00010000 0x0d088 0x0d088 R   0x10000
  LOAD           0x00d088 0x0002d088 0x0002d088 0xa09b8 0xa09b8 R E 0x10000
  LOAD           0x0ada40 0x000dda40 0x000dda40 0x0100c 0x0100c RW  0x10000
  LOAD           0x0aea50 0x000eea50 0x000eea50 0x00000 0x004a0 RW  0x10000
  TLS            0x0ada40 0x000cda40 0x000cda40 0x00000 0x00010 R   0x4
  DYNAMIC        0x0ae858 0x000de858 0x000de858 0x000c8 0x000c8 RW  0x4
  GNU_RELRO      0x0ada40 0x000dda40 0x000dda40 0x0100c 0x015c0 R   0x1
  GNU_STACK      0x000000 0x00000000 0x00000000 0x00000 0x1000000 RW  0
  NOTE           0x0001c8 0x000101c8 0x000101c8 0x00020 0x00020 R   0x4
  EXIDX          0x0016bc 0x000116bc 0x000116bc 0x00010 0x00010 R   0x4

I'm expecting the interpreter to be /lib/ld-linux-armhf.so.3, as I see on other C programs that are cross compiled. (as seen below)

> readelf -l ../config-tool/config-tool 

Elf file type is DYN (Shared object file)
Entry point 0xe5c
There are 9 program headers, starting at offset 52

Program Headers:
  Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
  EXIDX          0x001330 0x00001330 0x00001330 0x00008 0x00008 R   0x4
  PHDR           0x000034 0x00000034 0x00000034 0x00120 0x00120 R   0x4
  INTERP         0x000154 0x00000154 0x00000154 0x00019 0x00019 R   0x1
      [Requesting program interpreter: /lib/ld-linux-armhf.so.3]
  LOAD           0x000000 0x00000000 0x00000000 0x0133c 0x0133c R E 0x10000
  LOAD           0x001ef0 0x00011ef0 0x00011ef0 0x001b4 0x001b8 RW  0x10000
  DYNAMIC        0x001ef8 0x00011ef8 0x00011ef8 0x00108 0x00108 RW  0x4
  NOTE           0x000170 0x00000170 0x00000170 0x00020 0x00020 R   0x4
  GNU_STACK      0x000000 0x00000000 0x00000000 0x00000 0x00000 RW  0x10
  GNU_RELRO      0x001ef0 0x00011ef0 0x00011ef0 0x00110 0x00110 R   0x1

As I missing anything in cross compiling for my target platform?

Expected Behavior

Expected to be able to run program on the target (arm-linux-gnueabi).

[andrewrk]

it looks like you want gnueabihf (hard float).

zig/lib/std/target.zig

Lines 1582 to 1589 in 43b8304

	.arm,
	.armeb,
	.thumb,
	.thumbeb,
	=> return copy(&result, switch (self.abi.floatAbi()) {
	.hard => "/lib/ld-linux-armhf.so.3",
	else => "/lib/ld-linux.so.3",
	}),

[aka-mj]

That did help be get further but now see an illegal instruction error:

~> strace /home/midmark/vault/bin/wifiscan
execve("/home/midmark/vault/bin/wifiscan", ["/home/midmark/vault/bin/wifiscan"...], 0xbebabda0 /* 13 vars */) = 0
brk(NULL)                               = 0x421000
uname({sysname="Summit Linux", nodename="cm1sd", ...}) = 0
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/vfp/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l/vfp", 0xbe8261b0)  = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/v7l/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/v7l", 0xbe8261b0)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/vfp/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls/vfp", 0xbe8261b0)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/tls/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/tls", 0xbe8261b0)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/vfp/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l/vfp", 0xbe8261b0)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/v7l/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/v7l", 0xbe8261b0)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/vfp/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat64("/lib/vfp", 0xbe8261b0)          = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/libpthread.so.0", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\264N\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=125444, ...}) = 0
mmap2(NULL, 164456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6eff000
mprotect(0xb6f15000, 61440, PROT_NONE)  = 0
mmap2(0xb6f24000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0xb6f24000
mmap2(0xb6f26000, 4712, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6f26000
close(3)                                = 0
openat(AT_FDCWD, "/lib/libc.so.6", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\3\0\0\0\0\0\0\0\0\3\0(\0\1\0\0\0\234v\1\0004\0\0\0"..., 512) = 512
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb6f56000
fstat64(3, {st_mode=S_IFREG|0755, st_size=1254972, ...}) = 0
mmap2(NULL, 1323844, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb6dbb000
mprotect(0xb6ee9000, 65536, PROT_NONE)  = 0
mmap2(0xb6ef9000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12e000) = 0xb6ef9000
mmap2(0xb6efc000, 9028, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb6efc000
close(3)                                = 0
set_tls(0xb6f56d00)                     = 0
mprotect(0xb6ef9000, 8192, PROT_READ)   = 0
mprotect(0xb6f24000, 4096, PROT_READ)   = 0
mprotect(0xdd000, 8192, PROT_READ)      = 0
mprotect(0xb6f58000, 4096, PROT_READ)   = 0
set_tid_address(0xb6f568a8)             = 521
set_robust_list(0xb6f568b0, 12)         = 0
--- SIGILL {si_signo=SIGILL, si_code=ILL_ILLOPC, si_addr=0xcc83c} ---
+++ killed by SIGILL +++

A look at the ELF from the host machine:

~/zig/workspace/wifiscan> readelf -l zig-out/bin/wifiscan

Elf file type is EXEC (Executable file)
Entry point 0x2d090
There are 12 program headers, starting at offset 52

Program Headers:
  Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
  PHDR           0x000034 0x00010034 0x00010034 0x00180 0x00180 R   0x4
  INTERP         0x0001b4 0x000101b4 0x000101b4 0x00019 0x00019 R   0x1
      [Requesting program interpreter: /lib/ld-linux-armhf.so.3]
  LOAD           0x000000 0x00010000 0x00010000 0x0d090 0x0d090 R   0x10000
  LOAD           0x00d090 0x0002d090 0x0002d090 0xa0820 0xa0820 R E 0x10000
  LOAD           0x0ad8b0 0x000dd8b0 0x000dd8b0 0x0100c 0x0100c RW  0x10000
  LOAD           0x0ae8c0 0x000ee8c0 0x000ee8c0 0x00000 0x004a0 RW  0x10000
  TLS            0x0ad8b0 0x000cd8b0 0x000cd8b0 0x00000 0x00010 R   0x4
...

Any suggestions how to further troubleshoot the illegal instruction error?

[aka-mj]

I was able to create the issue using the default program created by zig init-exe and modifying the target in the build.zig file.

        .target = .{
            .cpu_arch = std.Target.Cpu.Arch.arm,
            .cpu_model = std.zig.CrossTarget.CpuModel{ .explicit = &std.Target.arm.cpu.cortex_a5 },
            .cpu_features_add = std.Target.arm.cpu.cortex_a5.features,
            .os_tag = std.Target.Os.Tag.linux,
            .abi = std.Target.Abi.gnueabihf,
        },

My cpu is a Microchip SAMA5D3 (Cortex-A5).

After capturing a core dump for this program and looking at it with GDB:

~/workspace/cm> gdb ./basesystem/midmark/vault/bin/armtest ./core.5135 
GNU gdb (GDB) 8.1.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "--host=x86_64-pc-linux-gnu --target=arm-buildroot-linux-gnueabihf".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from ./basesystem/midmark/vault/bin/armtest...done.
[New LWP 5135]
Core was generated by `./armtest'.
Program terminated with signal SIGILL, Illegal instruction.
#0  0x000991f0 in memset ()
(gdb) where
#0  0x000991f0 in memset ()
#1  0x0002dcac in os.linux.tls.prepareTLS (area=...) at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/os/linux/tls.zig:278
#2  0x0002c000 in os.linux.tls.initStaticTLS (phdrs=...) at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/os/linux/tls.zig:337
#3  0x0002b538 in start.posixCallMainAndExit () at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/start.zig:445
#4  0x0002b19c in _start () at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/start.zig:367
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) bt full
#0  0x000991f0 in memset ()
No symbol table info available.
#1  0x0002dcac in os.linux.tls.prepareTLS (area=...) at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/os/linux/tls.zig:278
        dtv = 0x0
        tcb_ptr = 0x20
#2  0x0002c000 in os.linux.tls.initStaticTLS (phdrs=...) at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/os/linux/tls.zig:337
        tls_area = {ptr = 0xab000 <os.linux.tls.main_thread_tls_buffer> "", len = 32}
        tp_value = 0
#3  0x0002b538 in start.posixCallMainAndExit () at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/start.zig:445
        auxv = 0xbee96de8
        at_hwcap = 123094
        phdrs = {ptr = 0x10034, len = 7}
        argv = 0xbee96da4
        argc = 1
        envp_optional = 0xbee96dac
        envp_count = 14
        envp = {ptr = 0xbee96dac, len = 14}
#4  0x0002b19c in _start () at /home/mjohn/zig/zig-linux-x86_64-0.11.0-dev.4271+1aacfa718/lib/std/start.zig:367
No locals.
Backtrace stopped: previous frame identical to this frame (corrupt stack?)

The zig code, the default program from zig init-exe:

const std = @import("std");

pub fn main() !void {
    // Prints to stderr (it's a shortcut based on `std.io.getStdErr()`)
    std.debug.print("All your {s} are belong to us.\n", .{"codebase"});

    // stdout is for the actual output of your application, for example if you
    // are implementing gzip, then only the compressed bytes should be sent to
    // stdout, not any debugging messages.
    const stdout_file = std.io.getStdOut().writer();
    var bw = std.io.bufferedWriter(stdout_file);
    const stdout = bw.writer();

    try stdout.print("Run `zig build test` to run the tests.\n", .{});

    try bw.flush(); // don't forget to flush!
}

test "simple test" {
    var list = std.ArrayList(i32).init(std.testing.allocator);
    defer list.deinit(); // try commenting this out and see if zig detects the memory leak!
    try list.append(42);
    try std.testing.expectEqual(@as(i32, 42), list.pop());
}

[ldube]

@aka-mj I am not sure if this is related. I see an illegal instruction exception on a freestanding sama5d3 project too. I build using zig build-exe -target arm-freestanding -mcpu=cortex_a5 sama5.zig My target is sama5d3-xplained.
Using JTAG I see that the board crashes when it encounters a VLDR (which is NEON instruction AFAIK). The SAMA5D3 FPU (VFPv4-D16) does not have NEON.

20206fe0:       ed5b0b02        vldr    d16, [fp, #-8]
20206fe4:       edc00b00        vstr    d16, [r0]

[aka-mj]

@ldube, interesting. I did further troubleshooting on my issue here:

1. compiler_rt not respecting the CPU features on cross-compile #16957
2. https://ziggit.dev/t/cross-compile-zig-fails-on-target-with-illegal-instruction/1505

It seems for me the issue may be that memset from zig is not getting replaced with the version from my lib C.

[alexrp]

@aka-mj if you're still able to reproduce this, can you please get the crash in GDB and do disas $pc-40,$pc+40 so we can figure out what instruction is actually causing the SIGILL?

[ldube]

Just for the record.
The undefined instructions in my freestanding project disappeared when I updated to version 0.14.0-dev.1651+ffd071f55
The previous version I used 0.14.0-dev.1511+54b668f8a still produced undefined instructions.

I build using this command:
zig build -Dtarget=arm-freestanding -Dcpu=cortex_a5+strict_align