From 158654a75940bf677f52567838036ef51b52c2c7 Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Wed, 10 May 2023 20:37:44 -0400 Subject: [PATCH 01/10] WIP httpd demo --- examples/http/httpd/.dockerignore | 3 ++ examples/http/httpd/Dockerfile | 69 ++++++++++++++++++++++++++++ examples/http/httpd/example_httpd.sh | 10 ++++ examples/http/httpd/harness_httpd.sh | 9 ++++ 4 files changed, 91 insertions(+) create mode 100644 examples/http/httpd/.dockerignore create mode 100644 examples/http/httpd/Dockerfile create mode 100755 examples/http/httpd/example_httpd.sh create mode 100755 examples/http/httpd/harness_httpd.sh diff --git a/examples/http/httpd/.dockerignore b/examples/http/httpd/.dockerignore new file mode 100644 index 00000000..1a60db3f --- /dev/null +++ b/examples/http/httpd/.dockerignore @@ -0,0 +1,3 @@ +httpd +example_httpd +example_httpd.o diff --git a/examples/http/httpd/Dockerfile b/examples/http/httpd/Dockerfile new file mode 100644 index 00000000..04f9a2f3 --- /dev/null +++ b/examples/http/httpd/Dockerfile @@ -0,0 +1,69 @@ +# Create a separate image with the latest source +FROM trailofbits/polytracker:latest +LABEL org.opencontainers.image.authors="lisa.overall@trailofbits.com" + +RUN rm -rf /polytracker/examples/http/httpd && mkdir -p /polytracker/examples/http/httpd + +WORKDIR /polytracker/examples/http/httpd +RUN git clone --branch 2.4.13 https://github.com/apache/httpd.git +RUN apt update && apt install -y curl autoconf libtool-bin && rm -rf /var/lib/apt/lists/* +# libapr1-dev libaprutil1-dev libpcre3-dev + +WORKDIR /polytracker/examples/http/httpd/httpd +RUN mkdir -p srclib/apr srclib/apr-util srclib/pcre srclib/expat +RUN curl https://archive.apache.org/dist/apr/apr-1.7.0.tar.gz -o apr-1.7.0.tar.gz \ + && tar xfz apr-1.7.0.tar.gz -C srclib/apr --strip-components 1 \ + && rm apr-1.7.0.tar.gz +RUN curl https://archive.apache.org/dist/apr/apr-util-1.6.1.tar.gz -o apr-util-1.6.1.tar.gz \ + && tar xfz apr-util-1.6.1.tar.gz -C srclib/apr-util --strip-components 1 \ + && rm apr-util-1.6.1.tar.gz +RUN curl -L https://sourceforge.net/projects/pcre/files/pcre/8.39/pcre-8.39.tar.gz/download -o pcre-8.39.tar.gz \ + && tar xfz pcre-8.39.tar.gz -C srclib/pcre --strip-components 1 \ + && rm pcre-8.39.tar.gz +RUN curl -L https://github.com/libexpat/libexpat/releases/download/R_2_4_7/expat-2.4.7.tar.gz -o expat-2.4.7.tar.gz \ + && tar xfz expat-2.4.7.tar.gz -C srclib/expat --strip-components 1 \ + && rm expat-2.4.7.tar.gz + +WORKDIR /polytracker/examples/http/httpd/httpd/srclib/pcre +RUN polytracker build ./configure --disable-shared +RUN polytracker build make +# NOTE: no longer needed after --disable-shared +RUN polytracker extract-bc -o ../../libpcre.bc .libs/libpcre.a + +WORKDIR /polytracker/examples/http/httpd/httpd/srclib/expat +RUN polytracker build ./configure --disable-shared +RUN polytracker build make +# NOTE: no longer needed after --disable-shared +RUN polytracker extract-bc -o ../../libexpat.bc lib/.libs/libexpat.a + +# apr, apr-util are configured via httpd's configure script + +WORKDIR /polytracker/examples/http/httpd/httpd +RUN polytracker build ./buildconf +# NOTE: CFLAGS reported unused during instrumentation +RUN CFLAGS="-I$(pwd)/srclib/pcre -I$(pwd)/srclib/expat/lib" \ +LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ + polytracker build ./configure --disable-shared --with-mpm=prefork --with-pcre=srclib/pcre/pcre-config --with-included-apr +# NOTE: CFLAGS reported unused during instrumentation +RUN CFLAGS="-I$(pwd)/srclib/pcre -I$(pwd)/srclib/expat/lib" \ +LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ + polytracker build make -j$((`nproc`+1)) + +# NOTE: no longer needed after configuring dependencies with --disable-shared +# RUN polytracker extract-bc -o httpd.bc httpd +# RUN llvm-link -o httpd-linked.bc httpd.bc libpcre.bc libexpat.bc +# RUN polytracker instrument-bc --taint --ftrace httpd-linked.bc -o instrumented.bc + +# TODO: fails here due to undefined references to getrandom() +# RUN polytracker instrument-targets --taint --ftrace httpd + +# NOTE: no longer needed after configuring dependencies with --disable-shared +# RUN polytracker lower-bc instrumented.bc -t httpd -o httpd.instrumented + +# RUN mv httpd.instrumented httpd_track + +COPY harness_httpd.sh /polytracker/examples/http/httpd/ + +# Note, the /workdir directory is intended to be mounted at runtime +VOLUME ["/workdir"] +#WORKDIR /workdir diff --git a/examples/http/httpd/example_httpd.sh b/examples/http/httpd/example_httpd.sh new file mode 100755 index 00000000..73ca2313 --- /dev/null +++ b/examples/http/httpd/example_httpd.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [[ "$(docker images -q trailofbits/polytracker 2>/dev/null)" == "" ]]; then + docker build -t trailofbits/polytracker -f ../../Dockerfile ../../ +fi +if [[ "$(docker images -q trailofbits/polytracker-demo-http-httpd 2>/dev/null)" == "" ]]; then + docker build -t trailofbits/polytracker-demo-http-httpd . +fi + +docker run --read-only -ti --rm -e POLYPATH="$1" --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest /polytracker/examples/http/httpd/harness_httpd.sh "$1" diff --git a/examples/http/httpd/harness_httpd.sh b/examples/http/httpd/harness_httpd.sh new file mode 100755 index 00000000..dced637d --- /dev/null +++ b/examples/http/httpd/harness_httpd.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +httpd_track & +APACHE_PID=$! +# TODO: send request (in file - first command line arg) to httpd_track +kill $APACHE_PID +wait $APACHE_PID From 03587ac80b28ee66cf0a327d4dea8a6ea778b144 Mon Sep 17 00:00:00 2001 From: hbrodin <90325907+hbrodin@users.noreply.github.com> Date: Thu, 11 May 2023 09:09:57 +0200 Subject: [PATCH 02/10] Instruct DFSan to discard getrandom. Allows the instrumetation of httpd to succeed. --- examples/http/httpd/Dockerfile | 4 ++-- polytracker/custom_abi/dfsan_abilist.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/http/httpd/Dockerfile b/examples/http/httpd/Dockerfile index 04f9a2f3..6e523a08 100644 --- a/examples/http/httpd/Dockerfile +++ b/examples/http/httpd/Dockerfile @@ -55,12 +55,12 @@ LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ # RUN polytracker instrument-bc --taint --ftrace httpd-linked.bc -o instrumented.bc # TODO: fails here due to undefined references to getrandom() -# RUN polytracker instrument-targets --taint --ftrace httpd +RUN polytracker instrument-targets --taint --ftrace httpd # NOTE: no longer needed after configuring dependencies with --disable-shared # RUN polytracker lower-bc instrumented.bc -t httpd -o httpd.instrumented -# RUN mv httpd.instrumented httpd_track +RUN mv httpd.instrumented httpd_track COPY harness_httpd.sh /polytracker/examples/http/httpd/ diff --git a/polytracker/custom_abi/dfsan_abilist.txt b/polytracker/custom_abi/dfsan_abilist.txt index 6ddf8a97..a3c5b55c 100644 --- a/polytracker/custom_abi/dfsan_abilist.txt +++ b/polytracker/custom_abi/dfsan_abilist.txt @@ -25,6 +25,8 @@ fun:open=uninstrumented fun:open=custom fun:open64=uninstrumented fun:open64=custom +fun:getrandom=uninstrumented +fun:getrandom=discard ########################################## From 5328c3ffa49dc6dfa4e8508d2ffc4154889b6767 Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Thu, 11 May 2023 11:37:04 -0400 Subject: [PATCH 03/10] Supply required configuration and modules for bare-bones server, complete harness --- examples/http/httpd/Dockerfile | 33 +-- examples/http/httpd/README.md | 29 ++ examples/http/httpd/example_httpd.sh | 3 +- examples/http/httpd/harness_httpd.sh | 18 +- examples/http/httpd/httpd.conf | 419 +++++++++++++++++++++++++++ 5 files changed, 473 insertions(+), 29 deletions(-) create mode 100644 examples/http/httpd/README.md create mode 100644 examples/http/httpd/httpd.conf diff --git a/examples/http/httpd/Dockerfile b/examples/http/httpd/Dockerfile index 6e523a08..7642d41b 100644 --- a/examples/http/httpd/Dockerfile +++ b/examples/http/httpd/Dockerfile @@ -1,4 +1,3 @@ -# Create a separate image with the latest source FROM trailofbits/polytracker:latest LABEL org.opencontainers.image.authors="lisa.overall@trailofbits.com" @@ -6,8 +5,7 @@ RUN rm -rf /polytracker/examples/http/httpd && mkdir -p /polytracker/examples/ht WORKDIR /polytracker/examples/http/httpd RUN git clone --branch 2.4.13 https://github.com/apache/httpd.git -RUN apt update && apt install -y curl autoconf libtool-bin && rm -rf /var/lib/apt/lists/* -# libapr1-dev libaprutil1-dev libpcre3-dev +RUN apt update && apt install -y netcat curl autoconf libtool-bin && rm -rf /var/lib/apt/lists/* WORKDIR /polytracker/examples/http/httpd/httpd RUN mkdir -p srclib/apr srclib/apr-util srclib/pcre srclib/expat @@ -27,43 +25,32 @@ RUN curl -L https://github.com/libexpat/libexpat/releases/download/R_2_4_7/expat WORKDIR /polytracker/examples/http/httpd/httpd/srclib/pcre RUN polytracker build ./configure --disable-shared RUN polytracker build make -# NOTE: no longer needed after --disable-shared -RUN polytracker extract-bc -o ../../libpcre.bc .libs/libpcre.a WORKDIR /polytracker/examples/http/httpd/httpd/srclib/expat RUN polytracker build ./configure --disable-shared RUN polytracker build make -# NOTE: no longer needed after --disable-shared -RUN polytracker extract-bc -o ../../libexpat.bc lib/.libs/libexpat.a # apr, apr-util are configured via httpd's configure script WORKDIR /polytracker/examples/http/httpd/httpd RUN polytracker build ./buildconf -# NOTE: CFLAGS reported unused during instrumentation RUN CFLAGS="-I$(pwd)/srclib/pcre -I$(pwd)/srclib/expat/lib" \ -LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ - polytracker build ./configure --disable-shared --with-mpm=prefork --with-pcre=srclib/pcre/pcre-config --with-included-apr -# NOTE: CFLAGS reported unused during instrumentation + LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ + polytracker build ./configure --disable-shared --with-mpm=prefork --with-pcre=srclib/pcre/pcre-config --with-included-apr \ + --enable-mods-static='authz_core unixd' RUN CFLAGS="-I$(pwd)/srclib/pcre -I$(pwd)/srclib/expat/lib" \ -LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ + LDFLAGS="-L$(pwd)/srclib/pcre/.libs -L$(pwd)/srclib/expat/lib/.libs" \ polytracker build make -j$((`nproc`+1)) -# NOTE: no longer needed after configuring dependencies with --disable-shared -# RUN polytracker extract-bc -o httpd.bc httpd -# RUN llvm-link -o httpd-linked.bc httpd.bc libpcre.bc libexpat.bc -# RUN polytracker instrument-bc --taint --ftrace httpd-linked.bc -o instrumented.bc - -# TODO: fails here due to undefined references to getrandom() RUN polytracker instrument-targets --taint --ftrace httpd - -# NOTE: no longer needed after configuring dependencies with --disable-shared -# RUN polytracker lower-bc instrumented.bc -t httpd -o httpd.instrumented - RUN mv httpd.instrumented httpd_track +# overwrite binary to be installed with our polytracker-instrumented version +RUN cp httpd_track httpd +RUN polytracker build make install COPY harness_httpd.sh /polytracker/examples/http/httpd/ +COPY httpd.conf /usr/local/apache2/conf/ # Note, the /workdir directory is intended to be mounted at runtime VOLUME ["/workdir"] -#WORKDIR /workdir +WORKDIR /workdir diff --git a/examples/http/httpd/README.md b/examples/http/httpd/README.md new file mode 100644 index 00000000..737d74fd --- /dev/null +++ b/examples/http/httpd/README.md @@ -0,0 +1,29 @@ +# Polytracker demo: Apache httpd + +## Quickstart +``` +cd /path/to/polytracker/examples/http/httpd +./example_httpd.sh foo.txt +``` +where `foo.txt` contains the raw text of an HTTP request. + +## Notes on instrumentation +In order to enable polytracker instrumentation, we statically compile httpd, its dependencies, and its modules. + +The default build includes the following statically compiled modules: +``` +$ ./httpd -l +Compiled in modules: + core.c + mod_authz_core.c + mod_so.c + http_core.c + prefork.c + mod_unixd.c +``` + +In order to enable additional modules, modify the Dockerfile to include additional `--enable-MODULE=static` +or `--enable-modules-static=MODULE-LIST` directives during the final `./configure` command. +(see the [httpd configruation documenation](https://httpd.apache.org/docs/2.4/programs/configure.html) for further details). +You may also need to modify the `httpd.conf` file in this directory (which is copied to `/usr/local/apache2/conf/httpd.conf` in the container), and potentially add module configuration files to +the `/usr/local/apache2/conf/extra` directory. diff --git a/examples/http/httpd/example_httpd.sh b/examples/http/httpd/example_httpd.sh index 73ca2313..27acaca0 100755 --- a/examples/http/httpd/example_httpd.sh +++ b/examples/http/httpd/example_httpd.sh @@ -7,4 +7,5 @@ if [[ "$(docker images -q trailofbits/polytracker-demo-http-httpd 2>/dev/null)" docker build -t trailofbits/polytracker-demo-http-httpd . fi -docker run --read-only -ti --rm -e POLYPATH="$1" --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest /polytracker/examples/http/httpd/harness_httpd.sh "$1" +# NOTE: cannot pass --read-only because httpd needs to be able to write to /usr/local/apache2/logs/error_log +docker run -ti --rm -e POLYPATH="$1" --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest /polytracker/examples/http/httpd/harness_httpd.sh "$1" diff --git a/examples/http/httpd/harness_httpd.sh b/examples/http/httpd/harness_httpd.sh index dced637d..0caec897 100755 --- a/examples/http/httpd/harness_httpd.sh +++ b/examples/http/httpd/harness_httpd.sh @@ -2,8 +2,16 @@ set -e -httpd_track & -APACHE_PID=$! -# TODO: send request (in file - first command line arg) to httpd_track -kill $APACHE_PID -wait $APACHE_PID +APACHE_ROOT=/usr/local/apache2 +$APACHE_ROOT/bin/apachectl -k start + +# send request (from text file - first command line arg) to instrumented httpd +nc localhost 80 <"$1" + +# alternatively: +# APACHE_PID=$(cat /usr/local/apache2/logs/httpd.pid) +# kill $APACHE_PID +# wait $APACHE_PID + +# alternatively: graceful-stop, in which currently open connections are not aborted +$APACHE_ROOT/bin/apachectl -k stop diff --git a/examples/http/httpd/httpd.conf b/examples/http/httpd/httpd.conf new file mode 100644 index 00000000..4b18e597 --- /dev/null +++ b/examples/http/httpd/httpd.conf @@ -0,0 +1,419 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot /usr/local/apache2 + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +Listen 127.0.0.1:80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +# @@LoadModule@@ + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User daemon +Group daemon + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +ServerName localhost + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride None + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog "logs/error_log" + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog "logs/access_log" common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + +# +# uncomment out the below to deal with user agents that deliberately +# violate open standards by misusing DNT (DNT *must* be a specific +# end-user choice) +# +# +#BrowserMatch "MSIE 10.0;" bad_DNT +# +# +#RequestHeader unset DNT env=bad_DNT +# + From f9a48028fcfcf72b900cddce047230678b21c8d7 Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Mon, 15 May 2023 07:51:59 -0400 Subject: [PATCH 04/10] Harness robust to HTTP requests located anywhere on host file system --- examples/http/httpd/example_httpd.sh | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/examples/http/httpd/example_httpd.sh b/examples/http/httpd/example_httpd.sh index 27acaca0..767f77db 100755 --- a/examples/http/httpd/example_httpd.sh +++ b/examples/http/httpd/example_httpd.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +if [[ -z "$1" ]]; then + echo "Error: no arguments supplied" + echo "Usage: ./example_httpd.sh /path/to/raw_http_request" + exit 1 +fi + if [[ "$(docker images -q trailofbits/polytracker 2>/dev/null)" == "" ]]; then docker build -t trailofbits/polytracker -f ../../Dockerfile ../../ fi @@ -7,5 +13,20 @@ if [[ "$(docker images -q trailofbits/polytracker-demo-http-httpd 2>/dev/null)" docker build -t trailofbits/polytracker-demo-http-httpd . fi +HOST_PATH=$(realpath $1) +HOST_DIR=$(dirname "$HOST_PATH") + # NOTE: cannot pass --read-only because httpd needs to be able to write to /usr/local/apache2/logs/error_log -docker run -ti --rm -e POLYPATH="$1" --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest /polytracker/examples/http/httpd/harness_httpd.sh "$1" + +# mount the file if it's not already in /workdir +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +if [[ "$HOST_DIR" == "$SCRIPT_DIR" ]]; then + docker run -ti --rm -e POLYPATH="$1" -e POLYDB="$1.tdag" \ + --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest \ + /polytracker/examples/http/httpd/harness_httpd.sh "$1" +else + CONTAINER_PATH=/workdir/$(basename "$1") + docker run -ti --rm -v "$HOST_PATH":"$CONTAINER_PATH" -e POLYPATH="$CONTAINER_PATH" -e POLYDB="$CONTAINER_PATH.tdag" \ + --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest \ + /polytracker/examples/http/httpd/harness_httpd.sh "$CONTAINER_PATH" +fi From 65fba32a2d12b6f641649bb2bbdef7f35b3a7c9a Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Mon, 15 May 2023 10:34:21 -0400 Subject: [PATCH 05/10] Use bind mount instead of named volume so docker cleans up appropriately --- examples/http/httpd/Dockerfile | 4 ++-- examples/http/httpd/example_httpd.sh | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/http/httpd/Dockerfile b/examples/http/httpd/Dockerfile index 7642d41b..8c250356 100644 --- a/examples/http/httpd/Dockerfile +++ b/examples/http/httpd/Dockerfile @@ -51,6 +51,6 @@ RUN polytracker build make install COPY harness_httpd.sh /polytracker/examples/http/httpd/ COPY httpd.conf /usr/local/apache2/conf/ -# Note, the /workdir directory is intended to be mounted at runtime -VOLUME ["/workdir"] +# Note, the /workdir and /testcase directories are intended to be mounted at runtime +VOLUME ["/workdir", "/testcase"] WORKDIR /workdir diff --git a/examples/http/httpd/example_httpd.sh b/examples/http/httpd/example_httpd.sh index 767f77db..3aaf69da 100755 --- a/examples/http/httpd/example_httpd.sh +++ b/examples/http/httpd/example_httpd.sh @@ -13,7 +13,8 @@ if [[ "$(docker images -q trailofbits/polytracker-demo-http-httpd 2>/dev/null)" docker build -t trailofbits/polytracker-demo-http-httpd . fi -HOST_PATH=$(realpath $1) +HOST_PATH=$(realpath "$1") +BASENAME=$(basename "$HOST_PATH") HOST_DIR=$(dirname "$HOST_PATH") # NOTE: cannot pass --read-only because httpd needs to be able to write to /usr/local/apache2/logs/error_log @@ -25,8 +26,10 @@ if [[ "$HOST_DIR" == "$SCRIPT_DIR" ]]; then --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest \ /polytracker/examples/http/httpd/harness_httpd.sh "$1" else - CONTAINER_PATH=/workdir/$(basename "$1") - docker run -ti --rm -v "$HOST_PATH":"$CONTAINER_PATH" -e POLYPATH="$CONTAINER_PATH" -e POLYDB="$CONTAINER_PATH.tdag" \ - --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker-demo-http-httpd:latest \ + CONTAINER_PATH=/testcase/"$BASENAME" + docker run -ti --rm -e POLYPATH="$CONTAINER_PATH" -e POLYDB=/workdir/"$BASENAME".tdag \ + --mount type=bind,source="$(pwd)",target=/workdir \ + --mount type=bind,source="$HOST_PATH",target="$CONTAINER_PATH" \ + trailofbits/polytracker-demo-http-httpd:latest \ /polytracker/examples/http/httpd/harness_httpd.sh "$CONTAINER_PATH" fi From a5a0466f3021b3213544451b4577a24ad2f25211 Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Tue, 16 May 2023 14:14:05 -0400 Subject: [PATCH 06/10] Instrument accept4, non-standard Linux extension used in httpd --- polytracker/src/taint_sources/taint_sources.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/polytracker/src/taint_sources/taint_sources.cpp b/polytracker/src/taint_sources/taint_sources.cpp index 4d4ea846..34e3bc81 100644 --- a/polytracker/src/taint_sources/taint_sources.cpp +++ b/polytracker/src/taint_sources/taint_sources.cpp @@ -466,6 +466,23 @@ EXT_C_FUNC int __dfsw_accept(int socket, struct sockaddr *address, return client_socket; } +EXT_C_FUNC int __dfsw_accept4(int socket, struct sockaddr *address, + socklen_t *address_len, int flags, + dfsan_label socket_label, + dfsan_label address_label, + dfsan_label address_len_label, + dfsan_label *ret_label) { + int client_socket = accept4(socket, address, address_len, flags); + if (client_socket >= 0) { + if (auto name = connect_name(client_socket); name) { + get_polytracker_tdag().open_file(client_socket, *name); + } + } + + *ret_label = 0; + return client_socket; +} + EXT_C_FUNC int __dfsw_connect(int socket, const struct sockaddr *address, socklen_t address_len, dfsan_label socket_label, dfsan_label address_label, From 7c041c2445032bba8553253b549c7a552fa91c86 Mon Sep 17 00:00:00 2001 From: hbrodin <90325907+hbrodin@users.noreply.github.com> Date: Tue, 16 May 2023 21:55:09 +0200 Subject: [PATCH 07/10] Tiny changes to instrument accept4 correctly --- polytracker/custom_abi/dfsan_abilist.txt | 1 + polytracker/src/taint_sources/taint_sources.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/polytracker/custom_abi/dfsan_abilist.txt b/polytracker/custom_abi/dfsan_abilist.txt index a3c5b55c..62176f43 100644 --- a/polytracker/custom_abi/dfsan_abilist.txt +++ b/polytracker/custom_abi/dfsan_abilist.txt @@ -1687,6 +1687,7 @@ fun:abs=functional fun:accept=uninstrumented fun:accept=custom fun:accept4=uninstrumented +fun:accept4=custom fun:access=uninstrumented fun:acct=uninstrumented fun:acos=uninstrumented diff --git a/polytracker/src/taint_sources/taint_sources.cpp b/polytracker/src/taint_sources/taint_sources.cpp index 34e3bc81..fba84587 100644 --- a/polytracker/src/taint_sources/taint_sources.cpp +++ b/polytracker/src/taint_sources/taint_sources.cpp @@ -471,6 +471,7 @@ EXT_C_FUNC int __dfsw_accept4(int socket, struct sockaddr *address, dfsan_label socket_label, dfsan_label address_label, dfsan_label address_len_label, + dfsan_label flags_label, dfsan_label *ret_label) { int client_socket = accept4(socket, address, address_len, flags); if (client_socket >= 0) { From deffb5b355d90b683e3413aaa0bda713dea6b00c Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Wed, 17 May 2023 11:16:49 -0400 Subject: [PATCH 08/10] Harness modifications to address console- and timing-related issues --- examples/http/httpd/harness_httpd.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/http/httpd/harness_httpd.sh b/examples/http/httpd/harness_httpd.sh index 0caec897..0dc79873 100755 --- a/examples/http/httpd/harness_httpd.sh +++ b/examples/http/httpd/harness_httpd.sh @@ -3,15 +3,18 @@ set -e APACHE_ROOT=/usr/local/apache2 -$APACHE_ROOT/bin/apachectl -k start +$APACHE_ROOT/bin/apachectl -X & +# needed for server initialization in single-worker mode +sleep 10 # send request (from text file - first command line arg) to instrumented httpd nc localhost 80 <"$1" -# alternatively: -# APACHE_PID=$(cat /usr/local/apache2/logs/httpd.pid) -# kill $APACHE_PID -# wait $APACHE_PID +APACHE_PID=$(cat /usr/local/apache2/logs/httpd.pid) +kill "$APACHE_PID" +wait -# alternatively: graceful-stop, in which currently open connections are not aborted -$APACHE_ROOT/bin/apachectl -k stop +# Oddly, these cause issues with TDAG production and does not include socket fds among TDAG sources +# but only when run from the same terminal +# $APACHE_ROOT/bin/apachectl stop +# $APACHE_ROOT/bin/apachectl graceful-stop From a807ce3952a2a725b5c4213c99c8bb54d7f091f4 Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Wed, 13 Sep 2023 10:44:06 -0400 Subject: [PATCH 09/10] Helper scripts to produce TDAGs for a collection of requests under every instrumented HTTP parser --- examples/http/faw_tdag_pairs.sh | 6 +++++ examples/http/tdag_pairs.sh | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 examples/http/faw_tdag_pairs.sh create mode 100755 examples/http/tdag_pairs.sh diff --git a/examples/http/faw_tdag_pairs.sh b/examples/http/faw_tdag_pairs.sh new file mode 100755 index 00000000..c9943dc1 --- /dev/null +++ b/examples/http/faw_tdag_pairs.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +FAW_DIR="$1" +./tdag_pairs.sh "$FAW_DIR"/cves +./tdag_pairs.sh "$FAW_DIR"/handcrafted +./tdag_pairs.sh "$FAW_DIR"/portswigger diff --git a/examples/http/tdag_pairs.sh b/examples/http/tdag_pairs.sh new file mode 100755 index 00000000..a2bc1674 --- /dev/null +++ b/examples/http/tdag_pairs.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -e + +if [[ -z "$1" ]]; then + echo "USAGE: ./tdag_pairs.sh /path/to/dir/with/raw_http_test_cases" + exit 1 +fi + +if [[ ! -d "$1" ]]; then + echo "ERROR: directory does not exist" + exit 2 +fi + +mkdir -p results +for file in "$1"/*; do + TEST_CASE=$(basename "$file") + if [[ ! -d results/"$TEST_CASE" ]]; then + mkdir -p results/"$TEST_CASE" + + # Could do this more elegantly by iterating over directories and excluding non-test-case dirs, + # e.g. via [[ $PARSER =~ ^(results)$ ]] && continue + # but there's a bunch of hidden directories with various code artifacts, + # which would be harder to maintain and could vary for different users + parser_array=("picohttpparser" "httpd") + for PARSER in "${parser_array[@]}"; do + echo "Producing TDAG for test case $TEST_CASE with parser $PARSER" + + # NOTE: if the instrumented process crashes, continue as long as we get a tdag + "$PARSER"/example_"$PARSER".sh "$file" || true + # TODO: use `polytracker compress` command once integrated + docker run --read-only -ti --rm --mount type=bind,source="$(pwd)",target=/workdir trailofbits/polytracker:latest \ + python3 /polytracker/examples/analysis/ubet/compress_tdag.py -i "$TEST_CASE".tdag -o "$TEST_CASE".tdag."$PARSER".compress + mv "$TEST_CASE".tdag."$PARSER".compress results/"$TEST_CASE" + rm "$TEST_CASE".tdag + done + + fi +done From 0973781b1913d4566b72a1263d44641ee1f454cd Mon Sep 17 00:00:00 2001 From: Lisa Overall <11458176+lisaoverall@users.noreply.github.com> Date: Wed, 13 Sep 2023 17:34:29 -0400 Subject: [PATCH 10/10] Configure httpd listening port via Docker environment variables --- examples/http/faw_tdag_pairs.sh | 6 ++++++ examples/http/httpd/example_httpd.sh | 16 ++++++++++++++-- examples/http/httpd/harness_httpd.sh | 12 ++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/examples/http/faw_tdag_pairs.sh b/examples/http/faw_tdag_pairs.sh index c9943dc1..f1d55775 100755 --- a/examples/http/faw_tdag_pairs.sh +++ b/examples/http/faw_tdag_pairs.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +if [[ -z "$1" ]]; then + echo "Error: no arguments supplied" + echo "Usage: ./example_httpd.sh /path/to/FAW/test_files/http" + exit 1 +fi + FAW_DIR="$1" ./tdag_pairs.sh "$FAW_DIR"/cves ./tdag_pairs.sh "$FAW_DIR"/handcrafted diff --git a/examples/http/httpd/example_httpd.sh b/examples/http/httpd/example_httpd.sh index 3aaf69da..7bf2d1dc 100755 --- a/examples/http/httpd/example_httpd.sh +++ b/examples/http/httpd/example_httpd.sh @@ -2,10 +2,22 @@ if [[ -z "$1" ]]; then echo "Error: no arguments supplied" - echo "Usage: ./example_httpd.sh /path/to/raw_http_request" + echo "Usage: ./example_httpd.sh /path/to/raw_http_request [httpd_port]" exit 1 fi +if [[ -z "$2" ]]; then + APACHE_PORT=80 +else + re='^[0-9]+$' + if ! [[ "$2" =~ $re ]] || [[ $2 -eq 0 ]] || [[ $2 -gt 65535 ]]; then + echo "Error: invalid httpd_port - must be positive integer in range 1-65535" + exit 1 + else + APACHE_PORT="$2" + fi +fi + if [[ "$(docker images -q trailofbits/polytracker 2>/dev/null)" == "" ]]; then docker build -t trailofbits/polytracker -f ../../Dockerfile ../../ fi @@ -27,7 +39,7 @@ if [[ "$HOST_DIR" == "$SCRIPT_DIR" ]]; then /polytracker/examples/http/httpd/harness_httpd.sh "$1" else CONTAINER_PATH=/testcase/"$BASENAME" - docker run -ti --rm -e POLYPATH="$CONTAINER_PATH" -e POLYDB=/workdir/"$BASENAME".tdag \ + docker run -ti --rm -e POLYPATH="$CONTAINER_PATH" -e POLYDB=/workdir/"$BASENAME".tdag -e APACHE_PORT="$APACHE_PORT" \ --mount type=bind,source="$(pwd)",target=/workdir \ --mount type=bind,source="$HOST_PATH",target="$CONTAINER_PATH" \ trailofbits/polytracker-demo-http-httpd:latest \ diff --git a/examples/http/httpd/harness_httpd.sh b/examples/http/httpd/harness_httpd.sh index 0dc79873..8cd05f9b 100755 --- a/examples/http/httpd/harness_httpd.sh +++ b/examples/http/httpd/harness_httpd.sh @@ -3,14 +3,22 @@ set -e APACHE_ROOT=/usr/local/apache2 + +# NB: this should be set via Docker if used with example_httpd.sh +if [[ -z "${APACHE_PORT}" ]]; then + APACHE_PORT=80 +else + sed -i 's/:80/:'"$APACHE_PORT"'/g' "$APACHE_ROOT"/conf/httpd.conf +fi + $APACHE_ROOT/bin/apachectl -X & # needed for server initialization in single-worker mode sleep 10 # send request (from text file - first command line arg) to instrumented httpd -nc localhost 80 <"$1" +nc localhost "$APACHE_PORT" <"$1" -APACHE_PID=$(cat /usr/local/apache2/logs/httpd.pid) +APACHE_PID=$(cat "$APACHE_ROOT"/logs/httpd.pid) kill "$APACHE_PID" wait