ASAN doesn't work #121

tycho · 2016-04-11T07:41:49Z

steven@localhost ~
$ cat test2.c
int main(int argc, char **argv)
{
        return 0;
}
steven@localhost ~
$ gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/usr/lib/gcc/x86_64-linux-gnu/4.8/lto-wrapper
Target: x86_64-linux-gnu
Configured with: ../src/configure -v --with-pkgversion='Ubuntu 4.8.5-2ubuntu1~14.04.1' --with-bugurl=file:///usr/share/doc/gcc-4.8/README.Bugs --enable-languages=c,c++,java,go,d,fortran,objc,obj-c++ --prefix=/usr --program-suffix=-4.8 --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --with-gxx-include-dir=/usr/include/c++/4.8 --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --enable-gnu-unique-object --disable-libmudflap --enable-plugin --with-system-zlib --disable-browser-plugin --enable-java-awt=gtk --enable-gtk-cairo --with-java-home=/usr/lib/jvm/java-1.5.0-gcj-4.8-amd64/jre --enable-java-home --with-jvm-root-dir=/usr/lib/jvm/java-1.5.0-gcj-4.8-amd64 --with-jvm-jar-dir=/usr/lib/jvm-exports/java-1.5.0-gcj-4.8-amd64 --with-arch-directory=amd64 --with-ecj-jar=/usr/share/java/eclipse-ecj.jar --enable-objc-gc --enable-multiarch --disable-werror --with-arch-32=i686 --with-abi=m64 --with-multilib-list=m32,m64,mx32 --with-tune=generic --enable-checking=release --build=x86_64-linux-gnu --host=x86_64-linux-gnu --target=x86_64-linux-gnu
Thread model: posix
gcc version 4.8.5 (Ubuntu 4.8.5-2ubuntu1~14.04.1)
steven@localhost ~
$ gcc -fsanitize=address -O0 -ggdb -o test2 test2.c
steven@localhost ~
$ ./test2
==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
    #0 0x7ff5fcc8231d==1910== AddressSanitizer CHECK failed: ../../../../src/libsanitizer/sanitizer_common/sanitizer_linux.cc:403 "((IsOnOf(*current_++, 's', 'p'))) != (0)" (0x0, 0x0)
^C
steven@localhost ~
$

Note that the ASAN printouts would have kept printing if I hadn't aborted with Ctrl-C.

The text was updated successfully, but these errors were encountered:

sunilmut · 2016-04-13T06:20:47Z

Thanks for reporting this. Could you grab a strace here to figure out what part is failing?

tycho · 2016-04-17T06:24:48Z

Looks related to #120.

278   munmap(0x7ff5fbd10000, 16384)     = 0
278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbd10000
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcd0000
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000
278   write(2, "    #0 0x7ff5fcc8231d", 21) = 21
278   munmap(0x7ff5fbcc0000, 16384)     = 0
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000
278   write(2, "==278== AddressSanitizer CHECK f"..., 162) = 162
278   munmap(0x7ff5fbcc0000, 16384)     = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbcc0000, 4096)      = 0
278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc1c0000-7ff5fc1c2000 rw-- 00000"..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbcc0000, 8192)      = 0
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc1c0000-7ff5fc1c2000 rw-- 00000"..., 4096) = 4096
278   read(3, "fda1000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1330
278   read(3, 0x7ff5fbcc2532, 4096)     = 0
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbcb0000, 4096)      = 0
278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc1b0000-7ff5fc1b1000 rw-- 00000"..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbcb0000, 8192)      = 0
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc1b0000-7ff5fc1b1000 rw-- 00000"..., 4096) = 4096
278   read(3, "fd94000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1378
278   read(3, 0x7ff5fbcb2562, 4096)     = 0
278   close(3)                          = 0
278   munmap(0x7ff5fbce0000, 16384)     = 0
278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbce0000
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbca0000
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000
278   write(2, "    #0 0x7ff5fcc8231d", 21) = 21
278   munmap(0x7ff5fbc90000, 16384)     = 0
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000
278   write(2, "==278== AddressSanitizer CHECK f"..., 162) = 162
278   munmap(0x7ff5fbc90000, 16384)     = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbc90000, 4096)      = 0
278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc190000-7ff5fc192000 rw-- 00000"..., 4096) = 4096
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   munmap(0x7ff5fbc90000, 8192)      = 0
278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   read(3, "fc190000-7ff5fc192000 rw-- 00000"..., 4096) = 4096
278   read(3, "fd71000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1474
278   read(3, 0x7ff5fbc925c2, 4096)     = 0
278   close(3)                          = 0
278   open("/proc/self/maps", O_RDONLY) = 3
278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc80000
278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096
278   --- SIGINT {si_signo=SIGINT, si_code=SI_KERNEL} ---                                                                                                                                                                                                                                   278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbd10000                                                                                                                                                                               278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcd0000                                                                                                                                                                               278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000                                                                                                                                                                              278   write(2, "    #0 0x7ff5fcc8231d", 21) = 21                                                                                                                                                                                                                              278   munmap(0x7ff5fbcc0000, 16384)     = 0                                                                                                                                                                                                                                   278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000                                                                                                                                                                              278   write(2, "==278== AddressSanitizer CHECK f"..., 162) = 162                                                                                                                                                                                                              278   munmap(0x7ff5fbcc0000, 16384)     = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbcc0000, 4096)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc1c0000-7ff5fc1c2000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbcc0000, 8192)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcc0000                                                                                                                                                                              278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc1c0000-7ff5fc1c2000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fda1000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1330                                                                                                                                                                                                            278   read(3, 0x7ff5fbcc2532, 4096)     = 0                                                                                                                                                                                                                                   278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbcb0000, 4096)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc1b0000-7ff5fc1b1000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbcb0000, 8192)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbcb0000                                                                                                                                                                              278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc1b0000-7ff5fc1b1000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fd94000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1378                                                                                                                                                                                                            278   read(3, 0x7ff5fbcb2562, 4096)     = 0                                                                                                                                                                                                                                   278   close(3)                          = 0                                                                                                                                                                                                                                   278   munmap(0x7ff5fbce0000, 16384)     = 0                                                                                                                                                                                                                                   278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbce0000                                                                                                                                                                               278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbca0000                                                                                                                                                                               278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000                                                                                                                                                                              278   write(2, "    #0 0x7ff5fcc8231d", 21) = 21                                                                                                                                                                                                                              278   munmap(0x7ff5fbc90000, 16384)     = 0                                                                                                                                                                                                                                   278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000                                                                                                                                                                              278   write(2, "==278== AddressSanitizer CHECK f"..., 162) = 162                                                                                                                                                                                                              278   munmap(0x7ff5fbc90000, 16384)     = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbc90000, 4096)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc190000-7ff5fc192000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   munmap(0x7ff5fbc90000, 8192)      = 0                                                                                                                                                                                                                                   278   mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc90000                                                                                                                                                                              278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fc190000-7ff5fc192000 rw-- 00000"..., 4096) = 4096                                                                                                                                                                                                             278   read(3, "fd71000 rw-- 00000000 00:00 0\n7f"..., 4096) = 1474                                                                                                                                                                                                            278   read(3, 0x7ff5fbc925c2, 4096)     = 0                                                                                                                                                                                                                                   278   close(3)                          = 0                                                                                                                                                                                                                                   278   open("/proc/self/maps", O_RDONLY) = 3                                                                                                                                                                                                                                   278   mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7ff5fbc80000                                                                                                                                                                               278   read(3, "00400000-00401000 r-x- 00000000 "..., 4096) = 4096                                                                                                                                                                                                             278   --- SIGINT {si_signo=SIGINT, si_code=SI_KERNEL} ---

It keeps reading /proc/self/maps in hoping -something- has changed, but I'm not sure what it's looking for.

Meinersbur · 2016-09-02T12:40:42Z

Entry for same bug in asan bug tracker: google/sanitizers#708

Still occurs on Windows 14393.105

kasper93 · 2017-04-20T21:03:44Z

Some information about the issue. ASAN expects procmaps to be formatted as follows
08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar
It fails on the last char of permissions part. It expects either s or p. On Windows information whether mapping is shared or private is not available and - is shown instead, which triggers assert here https://github.com/llvm-mirror/compiler-rt/blob/74c8f498fa128f0cd4eb97d603c017f250dd3694/lib/sanitizer_common/sanitizer_procmaps_linux.cc#L59

Can we expect this to be fixed? Seems like not a big bug to fix. Mapping need to be either private or shared so showing - is just silly.

Example of output on Windows:

$ cat /proc/self/maps
00400000-0040c000 r-x- 00000000 00:00 7623                       /bin/cat
0060b000-0060c000 r--- 0000b000 00:00 7623                       /bin/cat
0060c000-0060d000 rw-- 0000c000 00:00 7623                       /bin/cat
00bcb000-00bec000 rw-- 00000000 00:00 0                          [heap]
7f1976a30000-7f1976bef000 r-x- 00000000 00:00 202985             /lib/x86_64-linux-gnu/libc-2.23.so
7f1976bef000-7f1976bf8000 ---- 001bf000 00:00 202985             /lib/x86_64-linux-gnu/libc-2.23.so
7f1976bf8000-7f1976def000 ---- 00000000 00:00 0
7f1976def000-7f1976df3000 r--- 001bf000 00:00 202985             /lib/x86_64-linux-gnu/libc-2.23.so
7f1976df3000-7f1976df5000 rw-- 001c3000 00:00 202985             /lib/x86_64-linux-gnu/libc-2.23.so
7f1976df5000-7f1976df9000 rw-- 00000000 00:00 0
7f1976e00000-7f1976e26000 r-x- 00000000 00:00 203030             /lib/x86_64-linux-gnu/ld-2.23.so
7f1977000000-7f1977022000 rw-- 00000000 00:00 0
7f1977025000-7f1977026000 r--- 00025000 00:00 203030             /lib/x86_64-linux-gnu/ld-2.23.so
7f1977026000-7f1977027000 rw-- 00026000 00:00 203030             /lib/x86_64-linux-gnu/ld-2.23.so
7f1977027000-7f1977028000 rw-- 00000000 00:00 0
7f1977028000-7f19771c0000 r--- 00000000 00:00 22408              /usr/lib/locale/locale-archive
7f19771c0000-7f19771c1000 rw-- 00000000 00:00 0
7f19771d0000-7f19771d1000 rw-- 00000000 00:00 0
7f19771e0000-7f19771e1000 rw-- 00000000 00:00 0
7f19771f0000-7f19771f2000 rw-- 00000000 00:00 0
7ffffe41e000-7ffffec1e000 rw-- 00000000 00:00 0                  [stack]
7fffff2d8000-7fffff2d9000 r-x- 00000000 00:00 0                  [vdso]

kasper93 · 2017-04-21T21:23:32Z

I changed ASAN to accept Window's proc maps format. But it is still unusable. ASAN maps 20TB of virtual memory and it kills WSL. It is so slow that asan init code takes hours. Probably the same issue as #1671

MikeGitb · 2017-08-25T16:10:48Z

Would be great, if we could run ASAN (and the other sanitizers) on WSL.

firewave · 2017-10-20T12:06:17Z

Using 16299 it seems to get further now.

My code

#include <string.h>

int main() {
	char arr[2];
	strcpy(arr, "str");
	return 0;
}

Using the following command-line

gcc -fsanitize=address crash.c -o crash

Reports this

==3969==ERROR: AddressSanitizer failed to allocate 0xdfff0001000 (15392894357504) bytes at address 2008fff7000 (errno: 12)
==3969==ReserveShadowMemoryRange failed while trying to map 0xdfff0001000 bytes. Perhaps you're using ulimit -v
Aborted (core dumped)

The strange output:
strace.log

Seems like the interesting part

setrlimit(RLIMIT_CORE, {rlim_cur=0, rlim_max=0}) = 0
mmap(0x7fff7000, 268435456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7fff7000
mmap(0x2008fff7000, 15392894357504, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = -1 ENOMEM (Cannot allocate memory)

The same issue seems to apply using MemorySanitizer with clang:

==6442==ERROR: MemorySanitizer failed to allocate 0xf0000000000 (16492674416640) bytes at address 10000000000 (errno: 12)

Using

clang -fsanitize=memory crash.c -o crash

Both of these return immediately.

The problem with the long start-up because of mapping memory still seems to happen when using ThreadSanitizer (GCC and clang) though.

Using

gcc -fsanitize=thread crash.c -o crash

Or

clang -fsanitize=thread crash.c -o crash

zeux · 2017-10-30T04:59:12Z

Looking into this a bit, the behavior of mmap for large regions with MAP_NORESERVE is somewhat odd.

This code fails on my system with #if 0 but succeeds, although mmap & mprotect take tens of seconds each, with #if 1:

#include <sys/mman.h>
#include <stdio.h>

int main()
{
	size_t size = 0xdfff0001000;

#if 1
	void* ptr = mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
	printf("mmap => %p\n", ptr);
	int rc = mprotect(ptr, size, PROT_READ|PROT_WRITE);
	printf("mprotect => %d\n", rc);
#else
	void* ptr = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
	printf("mmap => %p\n", ptr);
#endif
}

As far as I can tell the memory mapped is completely usable after mprotect; it might be a reasonable workaround for asan even though it takes close to a minute to start up. This is on Windows 10 build 16299.19

bitcrazed · 2017-11-09T00:08:36Z

Hey all - suggestion from one of our devs: For the memory issue, try setting /proc/sys/vm/overcommit_memory to 1.

And do let us know if this helps!

/cc @MikeGitb

zeux · 2017-11-09T04:40:50Z

@bitcrazed Thanks! This fixes the mmap error and lets me run asan. At the end of the run (at least with my test program) I get this output: "LeakSanitizer has encountered a fatal error", but that's at exit so it's not as pressing.

A more significant issue now that asan startup is unblocked is performance; for my test program, ASAN run takes 19m30s and non-ASAN run takes 150ms. Valgrind run of a non-ASAN build takes 3 seconds.

zeux · 2017-11-09T05:18:05Z

Curiously, most of the time seems to be spent reading /proc/self/maps, not doing actual mmap; attaching strace output.

asan-trace.txt

firewave · 2017-11-09T11:58:52Z

It's the mmap after the reading of /proc/self/maps for me - so MSAN and ASAN binaries now start up and behave like TSAN ones. I only tested it with GCC.

kasper93 · 2017-11-09T18:25:18Z

Like I said few months ago, ASAN is affected by #1671 like many other tools that maps large amount of memory. Until that's fixed I doubt it will be usable.

kasper93 · 2017-12-19T22:37:38Z

FYI.

From issue #1671

We have improved mmap performance further in insider build 17063. I believe this makes stack ghc bearable to use now :).

This should significantly help sanitizers performance too.

zeux · 2017-12-22T19:07:41Z

With build 17063, the performance issues with asan seem to have been resolved - I still need to set overcommit_memory to 1, but after that it runs fine with good performance (~600ms vs ~150ms for non-asan run which seems reasonable).

There's still one issue at the end of execution:

==3742==Failed spawning a tracer thread (errno 22).
==3742==LeakSanitizer has encountered a fatal error.
==3742==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
==3742==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)

zeux · 2017-12-22T19:09:17Z

Running the program under strace (yeah I know the hint says "LeakSanitizer does not work under ptrace" :D) produces this syscall right before the fatal error:

clone(child_stack=0x7f3765120ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_UNTRACED) = -1 
EINVAL (Invalid argument)

Since errno 22 is EINVAL this might be the cause.

WSLUser · 2017-12-22T19:15:36Z

They fixed issues with both CLONE and EINVAL in previous Insider builds. I would submit a ticket to the package developer and get their input on it.

Edit: Unless something regressed due to the huge changes to 17063.

zeux · 2017-12-22T19:40:23Z

This program does reproduce the behavior (printing -1 on WSL, and a non-negative id on regular Linux):

#include <assert.h>
#include <sys/mman.h>
#include <unistd.h>
#include <sched.h>
#include <stdio.h>

int TracerThread(void *)
{
	return 0;
}

int main()
{
	const unsigned kStackSize = 2 * 1024 * 1024;
	void* stack = mmap(0, kStackSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
	pid_t tracer_pid = clone(TracerThread, (char*)stack + kStackSize, CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED, 0);
	printf("tracer pid: %d\n", int(tracer_pid));
}

So I would guess that this is a WSL issue with clone. I will file a separate issue for this.

therealkenc · 2017-12-22T23:58:11Z

The OP here was referring /proc/self/maps not being formatted correctly, described in #708 (comment) and addressed in 16215. Performance issues can be tracked in #1671. Running ASAN under strace can be tracked in #2781. That leaves MAP_NORESERVE AFAICT.

zeux · 2017-12-23T05:01:03Z

I filed MAP_NORESERVE issue separately as #2784.

therealkenc · 2018-02-08T19:06:14Z

If anyone who uses ASAN on a regular basis could give this a whirl again and see what (if anything) is blocking now that would be worth a go. #708, #2781, and #2784 have all been addressed in principle with 17093.

firewave · 2018-02-12T18:34:53Z

FYI I have to wait for it to hit the Slow Ring and will do so when it does.

firewave · 2018-03-14T21:33:46Z

Using 17115 and my previous example now works

=================================================================
==4724==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7ffffb45db52 at pc 0x7f872991c904 bp 0x7ffffb45db20 sp 0x7ffffb45d2c8
WRITE of size 4 at 0x7ffffb45db52 thread T0
    #0 0x7f872991c903 in __asan_memcpy (/usr/lib/x86_64-linux-gnu/libasan.so.2+0x8c903)
    #1 0x40094d in main (/home/sshuser/test+0x40094d)
    #2 0x7f87294e082f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2082f)
    #3 0x4007e8 in _start (/home/sshuser/test+0x4007e8)

Address 0x7ffffb45db52 is located in stack of thread T0 at offset 34 in frame
    #0 0x4008c5 in main (/home/sshuser/test+0x4008c5)

  This frame has 1 object(s):
    [32, 34) 'arr' <== Memory access at offset 34 overflows this variable
HINT: this may be a false positive if your program uses some custom stack unwind mechanism or swapcontext
      (longjmp and C++ exceptions *are* supported)
SUMMARY: AddressSanitizer: stack-buffer-overflow ??:0 __asan_memcpy
Shadow bytes around the buggy address:
  0x10007f683b10: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b20: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b30: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b40: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b50: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x10007f683b60: 00 00 00 00 00 00 f1 f1 f1 f1[02]f4 f4 f4 f3 f3
  0x10007f683b70: f3 f3 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683b90: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683ba0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x10007f683bb0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Heap right redzone:      fb
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack partial redzone:   f4
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
==4724==ABORTING

I also tested it with TSAN and MSAN as well as clang and well were fine. It was also fast in all cases without doing any changes to the system.

So I think this can now be closed. This is truly a great achievement.

therealkenc · 2018-03-15T02:04:03Z

Thanks. Calling this fixedinsiders 17093.

zeux · 2018-03-16T05:41:23Z

LeakSanitizer still fails for me in 17115 (in any program compiled with asan); the problem used to be in clone() - now clone() succeeds but prctl call after it fails:

clone(child_stack=0x7f09634f0ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_UNTRACED) = 595
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
getpid()                                = 594
prctl(PR_SET_PTRACER, 595)              = -1 EINVAL (Invalid argument)

The actual program output from Hello, world compiled with -fsanitize=address is:

/mnt/c/work/pugixml $ ./a.out
Hello, world!
==596==LeakSanitizer has encountered a fatal error.
==596==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
==596==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)

LeakSanitizer log:

/mnt/c/work/pugixml $ LSAN_OPTIONS=verbosity=1:log_threads=1 ./a.out
==598==AddressSanitizer: failed to intercept '__isoc99_printf'
==598==AddressSanitizer: failed to intercept '__isoc99_sprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_snprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_fprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_vprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_vsprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_vsnprintf'
==598==AddressSanitizer: failed to intercept '__isoc99_vfprintf'
==598==AddressSanitizer: failed to intercept '__cxa_throw'
==598==AddressSanitizer: libc interceptors initialized
|| `[0x10007fff8000, 0x7fffffffffff]` || HighMem    ||
|| `[0x02008fff7000, 0x10007fff7fff]` || HighShadow ||
|| `[0x00008fff7000, 0x02008fff6fff]` || ShadowGap  ||
|| `[0x00007fff8000, 0x00008fff6fff]` || LowShadow  ||
|| `[0x000000000000, 0x00007fff7fff]` || LowMem     ||
MemToShadow(shadow): 0x00008fff7000 0x000091ff6dff 0x004091ff6e00 0x02008fff6fff
redzone=16
max_redzone=2048
quarantine_size_mb=256M
malloc_context_size=30
SHADOW_SCALE: 3
SHADOW_GRANULARITY: 8
SHADOW_OFFSET: 0x7fff8000
==598==Installed the sigaction for signal 11
==598==Installed the sigaction for signal 7
==598==Installed the sigaction for signal 8
==598==T0: stack [0x7fffcea0e000,0x7fffcf20e000) size 0x800000; local=0x7fffcf20c774
==598==LeakSanitizer: Dynamic linker not found. TLS will not be handled correctly.
==598==AddressSanitizer Init done
Hello, world!
==599==Could not attach to thread 598 (errno 1).
==599==Failed suspending threads.
==598==LeakSanitizer has encountered a fatal error.
==598==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
==598==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)

Brian-Perkins · 2018-03-16T15:27:24Z

I would recommend opening a new issue for the PR_SET_PTRACER problem. If you set /proc/sys/kernel/yama/ptrace_scope to 0 does it start working? Looks like it is also mentioned in #2258 but not as a blocker.

zeux · 2018-03-26T04:06:55Z

Setting /proc/sys/kernel/yama/ptrace_scope to 0 works around the issue; I've filed #3053.

tara-raj · 2018-05-22T20:41:48Z

Fixed in Insiders Build 17093

21309582 · 2018-05-31T08:31:29Z

17134.81 and still not fixed as tara-raj claimed, but suggestion from zeux works.

echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope

This is using: gcc -fsanitize=address -g ./leaky.c && ./a.out

Edit: @therealkenc I just said it is not fixed in 1803, still need workaround. The tag "fixedin1803" is untrue.

thefallentree · 2018-11-21T07:07:52Z

must run
bash -c " echo 0 > /proc/sys/kernel/yama/ptrace_scope" for this to work under 1803

jackchammons added angular bug and removed angular labels Dec 13, 2016

abonander mentioned this issue Mar 4, 2017

Does not work on Windows Subsystem for Linux rust-fuzz/cargo-fuzz#55

Closed

MikeGitb mentioned this issue Nov 8, 2017

Location of WSL rootfs filesystem for post - Fall Creator's Update installations #2578

Closed

WSLUser mentioned this issue Dec 20, 2017

Issues with ASAN in Windows 10 Ubuntu subsystem google/sanitizers#708

Closed

zeux mentioned this issue Dec 22, 2017

clone() fails with CLONE_UNTRACED or without SIGCHLD #2781

Closed

zeux mentioned this issue Dec 23, 2017

large mmap() with MAP_NORESERVE fails unless overcommit_memory is 1 #2784

Closed

therealkenc added fixedininsiderbuilds and removed bug labels Mar 15, 2018

tara-raj closed this as completed May 22, 2018

therealkenc added fixedin1803 and removed fixedininsiderbuilds labels May 31, 2018

stoperro mentioned this issue Oct 23, 2018

LeakSanitizer leads to the program exiting with an error code with no output when leaks are present #3589

Closed

therealkenc mentioned this issue Aug 6, 2019

CONFIG_SECURITY_YAMA not set #4286

Closed

ginnedup mentioned this issue Nov 19, 2019

GDB reports exit code 01, program was supposed to return 0. #4691

Closed

therealkenc mentioned this issue Jun 9, 2020

enable CONFIG_SECURITY_YAMA=y microsoft/WSL2-Linux-Kernel#53

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ASAN doesn't work #121

ASAN doesn't work #121

tycho commented Apr 11, 2016

sunilmut commented Apr 13, 2016

tycho commented Apr 17, 2016 •

edited

Loading

Meinersbur commented Sep 2, 2016

kasper93 commented Apr 20, 2017 •

edited

Loading

kasper93 commented Apr 21, 2017 •

edited

Loading

MikeGitb commented Aug 25, 2017

firewave commented Oct 20, 2017 •

edited

Loading

zeux commented Oct 30, 2017 •

edited

Loading

bitcrazed commented Nov 9, 2017 •

edited

Loading

zeux commented Nov 9, 2017 •

edited

Loading

zeux commented Nov 9, 2017

firewave commented Nov 9, 2017 •

edited

Loading

kasper93 commented Nov 9, 2017

kasper93 commented Dec 19, 2017 •

edited

Loading

zeux commented Dec 22, 2017

zeux commented Dec 22, 2017 •

edited

Loading

WSLUser commented Dec 22, 2017 •

edited

Loading

zeux commented Dec 22, 2017

therealkenc commented Dec 22, 2017 •

edited

Loading

zeux commented Dec 23, 2017

therealkenc commented Feb 8, 2018 •

edited

Loading

firewave commented Feb 12, 2018

firewave commented Mar 14, 2018

therealkenc commented Mar 15, 2018

zeux commented Mar 16, 2018

Brian-Perkins commented Mar 16, 2018

zeux commented Mar 26, 2018

tara-raj commented May 22, 2018

21309582 commented May 31, 2018 •

edited

Loading

thefallentree commented Nov 21, 2018

ASAN doesn't work #121

ASAN doesn't work #121

Comments

tycho commented Apr 11, 2016

sunilmut commented Apr 13, 2016

tycho commented Apr 17, 2016 • edited Loading

Meinersbur commented Sep 2, 2016

kasper93 commented Apr 20, 2017 • edited Loading

kasper93 commented Apr 21, 2017 • edited Loading

MikeGitb commented Aug 25, 2017

firewave commented Oct 20, 2017 • edited Loading

zeux commented Oct 30, 2017 • edited Loading

bitcrazed commented Nov 9, 2017 • edited Loading

zeux commented Nov 9, 2017 • edited Loading

zeux commented Nov 9, 2017

firewave commented Nov 9, 2017 • edited Loading

kasper93 commented Nov 9, 2017

kasper93 commented Dec 19, 2017 • edited Loading

zeux commented Dec 22, 2017

zeux commented Dec 22, 2017 • edited Loading

WSLUser commented Dec 22, 2017 • edited Loading

zeux commented Dec 22, 2017

therealkenc commented Dec 22, 2017 • edited Loading

zeux commented Dec 23, 2017

therealkenc commented Feb 8, 2018 • edited Loading

firewave commented Feb 12, 2018

firewave commented Mar 14, 2018

therealkenc commented Mar 15, 2018

zeux commented Mar 16, 2018

Brian-Perkins commented Mar 16, 2018

zeux commented Mar 26, 2018

tara-raj commented May 22, 2018

21309582 commented May 31, 2018 • edited Loading

thefallentree commented Nov 21, 2018

tycho commented Apr 17, 2016 •

edited

Loading

kasper93 commented Apr 20, 2017 •

edited

Loading

kasper93 commented Apr 21, 2017 •

edited

Loading

firewave commented Oct 20, 2017 •

edited

Loading

zeux commented Oct 30, 2017 •

edited

Loading

bitcrazed commented Nov 9, 2017 •

edited

Loading

zeux commented Nov 9, 2017 •

edited

Loading

firewave commented Nov 9, 2017 •

edited

Loading

kasper93 commented Dec 19, 2017 •

edited

Loading

zeux commented Dec 22, 2017 •

edited

Loading

WSLUser commented Dec 22, 2017 •

edited

Loading

therealkenc commented Dec 22, 2017 •

edited

Loading

therealkenc commented Feb 8, 2018 •

edited

Loading

21309582 commented May 31, 2018 •

edited

Loading