From f27fadb286a607ee56449ce95136bdde72842859 Mon Sep 17 00:00:00 2001 From: kaoudis Date: Fri, 28 Jun 2024 13:28:08 +0000 Subject: [PATCH 1/9] libjpeg: stable sources link --- examples/Dockerfile-libjpeg.demo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/Dockerfile-libjpeg.demo b/examples/Dockerfile-libjpeg.demo index 52b65705..39fcc192 100644 --- a/examples/Dockerfile-libjpeg.demo +++ b/examples/Dockerfile-libjpeg.demo @@ -3,7 +3,8 @@ FROM ubuntu:focal AS libjpeg-sources WORKDIR /polytracker/the_klondike/ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get -y upgrade && apt-get install -y wget -RUN wget http://jpegclub.org/reference/wp-content/uploads/2022/01/jpegsrc.v9e.tar.gz && tar xf jpegsrc.v9e.tar.gz +RUN wget https://www.ijg.org/files/jpegsrc.v9e.tar.gz && \ + tar xvf jpegsrc.v9e.tar.gz # Now, build the libjpeg image using previously downloaded source FROM trailofbits/polytracker:latest From b0896cbb1502c16f491ca6170e27358a9f6dc34c Mon Sep 17 00:00:00 2001 From: kaoudis Date: Fri, 28 Jun 2024 13:40:44 +0000 Subject: [PATCH 2/9] nitro: custom build system of nitro breaks if old python isnt available, so turn off old python check - I think our switching to python3.10 only results in this environmental difference --- examples/Dockerfile-nitro-nitf.demo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Dockerfile-nitro-nitf.demo b/examples/Dockerfile-nitro-nitf.demo index af2a5024..3a84b0d0 100644 --- a/examples/Dockerfile-nitro-nitf.demo +++ b/examples/Dockerfile-nitro-nitf.demo @@ -14,7 +14,7 @@ WORKDIR /polytracker/the_klondike/nitro/build RUN polytracker build cmake .. \ -DCMAKE_C_FLAGS="-w -D_POSIX_C_SOURCE=200809L -DCODA_OSS_NO_is_trivially_copyable" \ -DCMAKE_CXX_FLAGS="-w -D_POSIX_C_SOURCE=200809L -DCODA_OSS_NO_is_trivially_copyable" \ - -DCODA_BUILD_TESTS=OFF + -DCODA_BUILD_TESTS=OFF -DENABLE_PYTHON=OFF RUN polytracker build cmake --build . -j$((`nproc`+1)) --target show_nitf++ From 7e02a88167a6afb6bb649099ce64a3ebb8c3a44c Mon Sep 17 00:00:00 2001 From: kaoudis Date: Fri, 28 Jun 2024 14:08:59 +0000 Subject: [PATCH 3/9] png: add missing wget dependency --- examples/Dockerfile-png.demo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Dockerfile-png.demo b/examples/Dockerfile-png.demo index 78add4c2..7f0165fa 100644 --- a/examples/Dockerfile-png.demo +++ b/examples/Dockerfile-png.demo @@ -15,7 +15,7 @@ RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list #Update pkg-config/util-linux (needed for FontConfig) RUN apt update RUN apt install pkg-config uuid-dev gperf libtool \ - gettext autopoint autoconf -y + gettext autopoint autoconf wget -y RUN apt-get install python3-dev From 803161890a499045792574142c8795d8101031f4 Mon Sep 17 00:00:00 2001 From: kaoudis Date: Fri, 28 Jun 2024 14:40:29 +0000 Subject: [PATCH 4/9] xpdf: fix zlib location (stored old versions modev to /fossils on their hosting) --- examples/Dockerfile-xpdf.demo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Dockerfile-xpdf.demo b/examples/Dockerfile-xpdf.demo index 06ff134b..999e4f45 100644 --- a/examples/Dockerfile-xpdf.demo +++ b/examples/Dockerfile-xpdf.demo @@ -45,7 +45,7 @@ RUN make -j5 install WORKDIR /polytracker/the_klondike #zlib -RUN wget https://www.zlib.net/zlib-1.2.11.tar.gz +RUN wget https://www.zlib.net/fossils/zlib-1.2.11.tar.gz RUN tar -xzvf zlib-1.2.11.tar.gz WORKDIR zlib-1.2.11 RUN ./configure --prefix=/usr && make -j$(nproc) test && make -j$(nproc) install From 6f69fddea86a128d46ad98f617af643c26b438b0 Mon Sep 17 00:00:00 2001 From: kaoudis Date: Fri, 28 Jun 2024 14:44:18 +0000 Subject: [PATCH 5/9] qpdf: use ijg stable jpeg source url --- examples/Dockerfile-qpdf.demo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/Dockerfile-qpdf.demo b/examples/Dockerfile-qpdf.demo index e835898a..7a98a612 100644 --- a/examples/Dockerfile-qpdf.demo +++ b/examples/Dockerfile-qpdf.demo @@ -4,7 +4,7 @@ WORKDIR /polytracker/the_klondike ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y git wget RUN git clone --depth=1 --branch 11.5 https://github.com/qpdf/qpdf.git -RUN wget http://jpegclub.org/reference/wp-content/uploads/2022/01/jpegsrc.v9e.tar.gz && tar xf jpegsrc.v9e.tar.gz +RUN wget https://www.ijg.org/files/jpegsrc.v9e.tar.gz && tar xf jpegsrc.v9e.tar.gz # Now, build the qpdf image using previously downloaded source FROM trailofbits/polytracker:latest From 77aba5813d1cb2f4f874500b4fc96ce5ec9bc0cd Mon Sep 17 00:00:00 2001 From: kaoudis Date: Tue, 2 Jul 2024 20:46:23 +0000 Subject: [PATCH 6/9] file: fix example Dockerfile by checking out a tag from reasonably around the time when we made the Dockerfile, and building in the correct location --- examples/Dockerfile-file.demo | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/Dockerfile-file.demo b/examples/Dockerfile-file.demo index 211ee90b..46c48492 100644 --- a/examples/Dockerfile-file.demo +++ b/examples/Dockerfile-file.demo @@ -3,18 +3,18 @@ MAINTAINER Carson Harmon WORKDIR /polytracker/the_klondike RUN apt update -RUN apt-get install automake libtool make python zlib1g-dev git -y -RUN git clone https://github.com/file/file.git +RUN apt-get install automake libtool make zlib1g-dev git -y RUN echo "temp" > /PLACEHOLDER ENV POLYPATH=/PLACEHOLDER -#================================= -WORKDIR file -RUN autoreconf -f -i +WORKDIR /polytracker/the_klondike +RUN git clone https://github.com/file/file.git +WORKDIR /polytracker/the_klondike/file +RUN git fetch --tags && \ + git checkout tags/FILE5_41 +RUN autoreconf -fiv RUN ./configure --prefix=/polytracker/the_klondike/bin/ --disable-shared RUN polytracker build make -j$((`nproc`+1)) install - -WORKDIR /polytracker/the_klondike/bin/bin RUN polytracker instrument-targets --taint --ftrace file --ignore-lists libz RUN mv file.instrumented file_track From 314b46932b5888b2dc01175d30755a8ce7740c26 Mon Sep 17 00:00:00 2001 From: kaoudis Date: Wed, 3 Jul 2024 00:42:47 +0000 Subject: [PATCH 7/9] xpdf: make everything work with modern polytracker, but a bit bigger and uglier than I'd prefer --- examples/Dockerfile-xpdf.demo | 107 ++++++++++++++++------------------ 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/examples/Dockerfile-xpdf.demo b/examples/Dockerfile-xpdf.demo index 999e4f45..ecd4653a 100644 --- a/examples/Dockerfile-xpdf.demo +++ b/examples/Dockerfile-xpdf.demo @@ -1,10 +1,12 @@ FROM trailofbits/polytracker -MAINTAINER Carson Harmon +LABEL org.opencontainers.image.authors="carson.harmon@trailofbits.com,kelly.kaoudis@trailofbits.com" WORKDIR /polytracker/the_klondike -#Update pkg-config/util-linux (needed for FontConfig) -RUN apt update && apt install pkg-config \ +# Update pkg-config/util-linux (needed for FontConfig) +# qt5 is needed for xpdf to build happily. +RUN apt-get update && apt-get install -y \ + pkg-config \ uuid-dev \ gperf \ wget \ @@ -18,89 +20,80 @@ RUN apt update && apt install pkg-config \ cmake \ libfreetype6-dev \ libxcb-composite0-dev \ - libxml2-dev -y + libxml2-dev \ + qtbase5-dev -#RUN apt-get install python3-dev - -#Fontconfig requires some stuff? -#RUN apt install pkg-config \libasound2-dev libssl-dev cmake libfreetype6-dev libexpat1-dev libxcb-composite0-dev -y -#RUN apt install libxml2-dev -y +WORKDIR /polytracker/the_klondike +RUN wget https://dl.xpdfreader.com/old/xpdf-4.03.tar.gz +RUN tar -xvf xpdf-4.03.tar.gz +ENV build_dir=/polytracker/the_klondike/xpdf-4.03 #================================= -WORKDIR /polytracker/the_klondike +WORKDIR $build_dir -#FreeType http://www.linuxfromscratch.org/blfs/view/svn/general/freetype2.html +# FreeType http://www.linuxfromscratch.org/blfs/view/svn/general/freetype2.html RUN wget https://downloads.sourceforge.net/freetype/freetype-2.10.1.tar.xz RUN tar -xvf freetype-2.10.1.tar.xz -WORKDIR freetype-2.10.1 +WORKDIR $build_dir/freetype-2.10.1 -#Some linux from scratch magic +# Some linux from scratch magic RUN sed -ri "s:.*(AUX_MODULES.*valid):\1:" modules.cfg RUN sed -r "s:.*(#.*SUBPIXEL_RENDERING) .*:\1:" -i include/freetype/config/ftoption.h RUN ./configure --prefix=/usr --enable-freetype-config -RUN make -j5 install +RUN make -j$(nproc) install #================================= -WORKDIR /polytracker/the_klondike +WORKDIR $build_dir -#zlib +# zlib RUN wget https://www.zlib.net/fossils/zlib-1.2.11.tar.gz RUN tar -xzvf zlib-1.2.11.tar.gz -WORKDIR zlib-1.2.11 -RUN ./configure --prefix=/usr && make -j$(nproc) test && make -j$(nproc) install +WORKDIR $build_dir/zlib-1.2.11 +RUN ./configure --prefix=/usr && \ + make -j$(nproc) test && \ + make -j$(nproc) install #================================= -WORKDIR /polytracker/the_klondike +WORKDIR $build_dir -#Libxml2 +# Libxml2 -RUN wget http://xmlsoft.org/sources/libxml2-2.9.10.tar.gz -RUN tar -xvf libxml2-2.9.10.tar.gz -WORKDIR libxml2-2.9.10 +RUN wget http://xmlsoft.org/sources/libxml2-2.9.11.tar.gz +RUN tar -xvf libxml2-2.9.11.tar.gz +WORKDIR $build_dir/libxml2-2.9.11 RUN ./configure --disable-dependency-tracking --prefix=/usr --with-python=/usr/bin/python3 -RUN make -j5 install -RUN make install - +RUN make -j$(nproc) install #================================= -WORKDIR /polytracker/the_klondike +WORKDIR $build_dir -##Fontconfig (depends on FreeType), note that the linux from scratch version is broken -#The gitlab version is up to date, and has a PR merged from a year ago with the bug fix -#https://gitlab.freedesktop.org/fontconfig/fontconfig/merge_requests/2/diffs?commit_id=8208f99fa1676c42bfd8d74de3e9dac5366c150c +# Fontconfig (depends on FreeType), note that the linux from scratch version is broken +# The gitlab version is up to date, and has a PR merged from a year ago with the bug fix +# https://gitlab.freedesktop.org/fontconfig/fontconfig/merge_requests/2/diffs?commit_id=8208f99fa1676c42bfd8d74de3e9dac5366c150c RUN git clone https://gitlab.freedesktop.org/fontconfig/fontconfig.git -WORKDIR fontconfig +WORKDIR $build_dir/fontconfig RUN ./autogen.sh --sysconfdir=/etc --prefix=/usr --enable-libxml2 --mandir=/usr/share/man -RUN make -j5 install +RUN make -j$(nproc) install #================================= -WORKDIR /polytracker/the_klondike - -RUN wget https://dl.xpdfreader.com/xpdf-4.03.tar.gz -RUN tar -xvf xpdf-4.03.tar.gz -WORKDIR xpdf-4.03 -RUN mkdir build -WORKDIR build -RUN cmake -DCMAKE_BUILD_TYPE=Release .. -RUN make -j5 install +# build, and record the build with Blight +WORKDIR $build_dir/build +RUN polytracker build cmake -S .. -B . -DCMAKE_BUILD_TYPE=Release +RUN polytracker build make -j$(nproc) install #================================= -WORKDIR xpdf - -#Extract and instrument pdftotext, other poppler tools should work the same -RUN get-bc -b pdftotext -RUN ${CXX} --lower-bitcode -i pdftotext.bc -o pdftotext_track --libs /build_artifacts/libfofi.a /build_artifacts/libgoo.a /build_artifacts/libsplash.a freetype fontconfig pthread --lists freetype fontconfig xml2 libz - -RUN get-bc -b pdfinfo -RUN ${CXX} --lower-bitcode -i pdfinfo.bc -o pdfinfo_track --libs /build_artifacts/libfofi.a /build_artifacts/libgoo.a /build_artifacts/libsplash.a freetype fontconfig pthread --lists freetype fontconfig xml2 libz - - -RUN get-bc -b pdftops -RUN ${CXX} --lower-bitcode -i pdtops.bc -o pdftops_track --libs /build_artifacts/libfofi.a /build_artifacts/libgoo.a /build_artifacts/libsplash.a freetype fontconfig pthread --lists freetype fontconfig xml2 libz - -# Note, the /workdir directory is intended to be mounted at runtime -#VOLUME ["/workdir"] -#WORKDIR /workdir +# Extract and instrument each tool of interest +RUN polytracker extract-bc -o pdftotext.bc xpdf/pdftotext +RUN polytracker instrument-bc --taint --ftrace pdftotext.bc -o pdftotext.instrumented.bc --ignore-lists freetype fontconfig xml2 libz +RUN polytracker lower-bc pdftotext.instrumented.bc -t pdftotext -o pdftotext_track + +RUN polytracker extract-bc -o pdfinfo.bc xpdf/pdfinfo +RUN polytracker instrument-bc --taint --ftrace pdfinfo.bc -o pdfinfo.instrumented.bc --ignore-lists freetype fontconfig xml2 libz +RUN polytracker lower-bc pdfinfo.instrumented.bc -t pdfinfo -o pdfinfo_track + +RUN polytracker extract-bc -o pdftops.bc xpdf/pdftops +RUN polytracker instrument-bc --taint --ftrace pdftops.bc -o pdftops.instrumented.bc --ignore-lists freetype fontconfig xml2 libz +RUN polytracker lower-bc pdftops.instrumented.bc -t pdftops -o pdftops_track From 367eb7402e13e41cdaf06130f5ecf0d0db0ab534 Mon Sep 17 00:00:00 2001 From: kaoudis Date: Wed, 3 Jul 2024 00:57:29 +0000 Subject: [PATCH 8/9] xpdf: one big instrument-targets --- examples/Dockerfile-xpdf.demo | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/examples/Dockerfile-xpdf.demo b/examples/Dockerfile-xpdf.demo index ecd4653a..da6cbbe6 100644 --- a/examples/Dockerfile-xpdf.demo +++ b/examples/Dockerfile-xpdf.demo @@ -85,15 +85,13 @@ RUN polytracker build cmake -S .. -B . -DCMAKE_BUILD_TYPE=Release RUN polytracker build make -j$(nproc) install #================================= -# Extract and instrument each tool of interest -RUN polytracker extract-bc -o pdftotext.bc xpdf/pdftotext -RUN polytracker instrument-bc --taint --ftrace pdftotext.bc -o pdftotext.instrumented.bc --ignore-lists freetype fontconfig xml2 libz -RUN polytracker lower-bc pdftotext.instrumented.bc -t pdftotext -o pdftotext_track - -RUN polytracker extract-bc -o pdfinfo.bc xpdf/pdfinfo -RUN polytracker instrument-bc --taint --ftrace pdfinfo.bc -o pdfinfo.instrumented.bc --ignore-lists freetype fontconfig xml2 libz -RUN polytracker lower-bc pdfinfo.instrumented.bc -t pdfinfo -o pdfinfo_track - -RUN polytracker extract-bc -o pdftops.bc xpdf/pdftops -RUN polytracker instrument-bc --taint --ftrace pdftops.bc -o pdftops.instrumented.bc --ignore-lists freetype fontconfig xml2 libz -RUN polytracker lower-bc pdftops.instrumented.bc -t pdftops -o pdftops_track +# Extract and instrument each tool of interest. +# This should make three instrumented binaries (originals live in the xpdf/ directory): +# pdftops.instrumented, pdftotext.instrumented, and pdfinfo.instrumented +# These commands are split up for timing / debugging purposes but you could +# run them all as one big instrument-targets as well. +RUN polytracker instrument-targets --taint --ftrace pdftotext --ignore-lists freetype fontconfig xml2 libz + +RUN polytracker instrument-targets --taint --ftrace pdfinfo --ignore-lists freetype fontconfig xml2 libz + +RUN polytracker instrument-targets --taint --ftrace pdftops --ignore-lists freetype fontconfig xml2 libz \ No newline at end of file From 9f38008fc9f2580a4aa66565ec2218dbaa5b3cee Mon Sep 17 00:00:00 2001 From: kaoudis Date: Wed, 3 Jul 2024 14:40:13 +0000 Subject: [PATCH 9/9] xpdf: comments --- examples/Dockerfile-xpdf.demo | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/Dockerfile-xpdf.demo b/examples/Dockerfile-xpdf.demo index da6cbbe6..b6b440a4 100644 --- a/examples/Dockerfile-xpdf.demo +++ b/examples/Dockerfile-xpdf.demo @@ -3,8 +3,9 @@ LABEL org.opencontainers.image.authors="carson.harmon@trailofbits.com,kelly.kaou WORKDIR /polytracker/the_klondike -# Update pkg-config/util-linux (needed for FontConfig) -# qt5 is needed for xpdf to build happily. +# Updating pkg-config/util-linux is needed for FontConfig +# Note also that qt5 is needed for the cmake build of xpdf +# (there appears to not be a cmake macro to turn off the build parts that require it) RUN apt-get update && apt-get install -y \ pkg-config \ uuid-dev \