Merge pull request #3492 from michaelhixson/master

Put the docker branch into the master branch

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:16.04
+
+# One -q produces output suitable for logging (mostly hides
+# progress indicators)
+RUN apt-get -yq update
+
+# WARNING: DONT PUT A SPACE AFTER ANY BACKSLASH OR APT WILL BREAK
+RUN apt-get -qqy install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+  git-core \
+  cloc dstat                    `# Collect resource usage statistics` \
+  python-dev \
+  python-pip \
+  python-software-properties \
+  libmysqlclient-dev            `# Needed for MySQL-python` \
+  libpq-dev                     `# Needed for psycopg2`
+
+RUN pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo docker==3.1.0
+
+ENV PYTHONPATH /FrameworkBenchmarks
+ENV FWROOT /FrameworkBenchmarks
+
+ENTRYPOINT ["python", "/FrameworkBenchmarks/toolset/run-tests.py"]

README.md

@@ -9,7 +9,7 @@ If you're new to the project, welcome! Please feel free to ask questions [here](
 
 This project provides representative performance measures across a wide field of web application frameworks. With much help from the community, coverage is quite broad and we are happy to broaden it further with contributions. The project presently includes frameworks on many languages including `Go`, `Python`, `Java`, `Ruby`, `PHP`, `C#`, `Clojure`, `Groovy`, `Dart`, `JavaScript`, `Erlang`, `Haskell`, `Scala`, `Perl`, `Lua`, `C`, and others.  The current tests exercise plaintext responses, JSON seralization, database reads and writes via the object-relational mapper (ORM), collections, sorting, server-side templates, and XSS counter-measures. Future tests will exercise other components and greater computation.
 
-[Read more and see the results of our tests on cloud and physical hardware](http://www.techempower.com/benchmarks/). 
+[Read more and see the results of our tests on Amazon EC2 and physical hardware](http://www.techempower.com/benchmarks/). 
 For descriptions of the test types that we run, see the 
 [test requirements section](https://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/).
 
@@ -25,25 +25,45 @@ required.
 
         $ git clone https://github.com/TechEmpower/FrameworkBenchmarks.git
 
-2. Move into the vagrant-development directory.
+2. Change directories
 
         $ cd FrameworkBenchmarks/deployment/vagrant
 
-3. Turn on the VM (takes at least 20 minutes).
+3. Create the TFB Docker virtual network
 
-        $ vagrant up
+        $ docker network create tfb
 
-4. Enter the VM.
+4. Run a test.
 
-        $ vagrant ssh
+        $ docker run -it --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=[ABS PATH TO THIS DIR],target=/FrameworkBenchmarks techempower/tfb --mode verify --test gemini
 
-5. Move into the FrameworkBenchmarks directory in the vm.
+### Explanation of the run script
 
-        vagrant@TFB-all:~$ cd ~/FrameworkBenchmarks
-        
-6. Run a test.
+That run script is pretty wordy, but each and every flag is required. Unfortunately, because of the way that Docker runs processes, you **cannot** put this inside of a shell script without breaking how `ctrl+c` and `SIGTERM` work (the shell script would receive the signal, do nothing with the underlying python suite running, and exit, orphaning the toolset to continue running).
 
-        vagrant@TFB-all:~/FrameworkBenchmarks$ tfb --mode verify --test beego
+- `-it` tells docker to run this in 'interactive' mode and simulate a TTY, so that `ctrl+c` is propagated.
+- `--network=tfb` tells the container to join the 'tfb' Docker virtual network
+- `-v` specifies which Docker socket path to mount as a volume in the running container. This allows docker commands run inside this container to use the host container's docker to create/run/stop/remove containers.
+- `--mount` mounts the FrameworkBenchmarks source directory as a volume to share with the container so that rebuilding the toolset image is unnecessary and any changes you make on the host system are available in the running toolset container.
+- `techempower/tfb` is the name of toolset container to run
+- `--mode verify --test gemini` are the command to pass to the toolset.
+
+#### A note on Linux:
+
+You may not want to call step 4 from above every time. You can add an `alias` to your `~/.bash_aliases` file to shorten it since it will not change once configured:
+
+`$ alias tfb="docker run -it --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=[ABS PATH TO THIS DIR],target=/FrameworkBenchmarks techempower/tfb"`
+
+`$ source ~/.bash_aliases`
+
+Now you can run the toolset via `tfb`:
+
+`$ tfb --mode verify --test gemini`
+
+#### A note on Windows:
+
+- Docker expects Linux-style paths. If you cloned on your `C:\` drive, then `[ABS PATH TO THIS DIR]` would be `/c/FrameworkBenchmarks`.
+- [Docker for Windows](https://www.docker.com/docker-windows) understands `/var/run/docker.sock` even though that is not a valid path on Windows. [Docker Toolbox](https://docs.docker.com/toolbox/toolbox_install_windows/) **may** not - use at your own risk.
 
 ## Add a New Test
 
@@ -65,10 +85,10 @@ If you find any errors or areas for improvement within the docs, feel free to ei
 Results of continuous benchmarking runs are available in real time [here](https://tfb-status.techempower.com/).
 
 #### Data Visualization
-If you have a `results.json` file that you would like to visualize, you can [do that here](https://www.techempower.com/benchmarks/#section=test). You can also attach a `runid` parameter to that url where `runid` is a run listed on [tfb-status](https://tfb-status.techempower.com) like so: https://www.techempower.com/benchmarks/#section=test&runid=fd07b64e-47ce-411e-8b9b-b13368e988c6
+If you have a `results.json` file that you would like to visualize, you can [do that here](https://www.techempower.com/benchmarks/#section=test)(these will be visualized using the metadata from the last known round; if you are adding a new test, it will not visualize anything). You can also attach a `runid` parameter to that url where `runid` is a run listed on [tfb-status](https://tfb-status.techempower.com) like so: https://www.techempower.com/benchmarks/#section=test&runid=fd07b64e-47ce-411e-8b9b-b13368e988c6
 
 ## Contributing
 
 The community has consistently helped in making these tests better, and we welcome any and all changes. Reviewing our contribution practices and guidelines will help to keep us all on the same page. The [contribution guide](https://frameworkbenchmarks.readthedocs.org/en/latest/Development/Contributing-Guide/) can be found in the [TFB documentation](https://frameworkbenchmarks.readthedocs.org/).
 
-Join in the conversation on our [mailing list](https://groups.google.com/forum/?fromgroups=#!forum/framework-benchmarks), on [Twitter](https://twitter.com/tfbenchmarks), or chat with us on [Freenode](https://webchat.freenode.net/) at `#techempower-fwbm`. 
+Join in the conversation at our [Google Group](https://groups.google.com/forum/?fromgroups=#!forum/framework-benchmarks), or chat with us on [Freenode](https://webchat.freenode.net/) at `#techempower-fwbm`. 

deployment/vagrant/bootstrap.sh

@@ -9,68 +9,18 @@ if [ ! -e "~/.firstboot" ]; then
   echo "grub-pc grub-pc/install_devices multiselect     /dev/sda" | sudo debconf-set-selections
 
   # Install prerequisite tools
-  echo "Installing prerequisites"
+  echo "Installing docker"
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+  sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
   sudo apt-get update
-  sudo apt-get install -y git
-  git config --global core.autocrlf input
+  sudo apt-get install -yqq docker-ce
 
-  # Setting up ssh & passwordless sudo
-  ssh-keygen -f /home/vagrant/.ssh/id_rsa -N '' -t rsa
-  cat /home/vagrant/.ssh/id_rsa.pub >> /home/vagrant/.ssh/authorized_keys
-  chmod og-wx /home/vagrant/.ssh/authorized_keys
-  echo "NoHostAuthenticationForLocalhost yes" | tee -a /home/vagrant/.ssh/config
-  chmod 600 ~/.ssh/config
+  # Setting up passwordless sudo
   echo "vagrant ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
 
-  # Setting up hosts aliases
-  echo 127.0.0.1 TFB-database | sudo tee --append /etc/hosts
-  echo 127.0.0.1 TFB-client   | sudo tee --append /etc/hosts
-  echo 127.0.0.1 TFB-server   | sudo tee --append /etc/hosts
-
-  # Setting up new FWROOT
-  export FWROOT="/home/vagrant/FrameworkBenchmarks"
-  echo `export FWROOT="/home/vagrant/FrameworkBenchmarks"` >> ~/.bashrc
-
   sudo chown vagrant:vagrant ~/FrameworkBenchmarks
   cd ~/FrameworkBenchmarks
 
-  # Set up the benchmark.cfg for vagrant user
-cat <<EOF > benchmark.cfg
-[Defaults]
-# Available Keys:
-os=linux
-server_host=TFB-server
-client_host=TFB-client
-client_identity_file=/home/vagrant/.ssh/id_rsa
-client_user=vagrant
-database_host=TFB-database
-database_identity_file=/home/vagrant/.ssh/id_rsa
-database_os=linux
-database_user=vagrant
-duration=60
-exclude=None
-install=server
-install_error_action=continue
-install_strategy=unified
-install_only=False
-list_tests=False
-concurrency_levels=[8, 16, 32, 64, 128, 256]
-pipeline_concurrency_levels=[256,1024,4096,16384]
-query_levels=[1, 5,10,15,20]
-cached_query_levels=[1,10,20,50,100]
-threads=8
-mode=benchmark
-sleep=60
-test=None
-type=all
-verbose=True
-clean=False
-clean_all=False
-ulimit=200000
-EOF
-
-  source ./toolset/setup/linux/prerequisites.sh
-
   # Setup a nice welcome message for our guest
   echo "Setting up welcome message"
   sudo rm -f /etc/update-motd.d/51-cloudguest
@@ -79,9 +29,6 @@ EOF
   sudo cat <<EOF > motd
 Welcome to the FrameworkBenchmarks project!
 
-  To get started, perhaps try this:
-    $ cd FrameworkBenchmarks
-
   You can get lots of help:
     $ tfb --help
 
@@ -91,6 +38,11 @@ Welcome to the FrameworkBenchmarks project!
   This Vagrant environment is already setup and ready to go.
 EOF
 
-  sudo mv motd /etc/
+  cat <<EOF > /home/vagrant/.bash_aliases
+alias tfb="docker run -it --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=/home/vagrant/FrameworkBenchmarks,target=/FrameworkBenchmarks techempower/tfb"
+EOF
 
+  sudo mv motd /etc/
+  sudo chmod 777 /var/run/docker.sock
+  docker network create tfb
 fi

deployment/vagrant/core.rb

@@ -11,9 +11,8 @@ def provision_bootstrap(config)
 
 def provider_libvirt(config)
   config.vm.provider :libvirt do |virt, override|
-    virt.name = "TechEmpower Framework Benchmarks"
     override.vm.hostname = "TFB-all"
-    override.vm.box = "RX14/trusty64"
+    override.vm.box = "generic/ubuntu1604"
 
     unless ENV.fetch('TFB_SHOW_VM', false)
       virt.graphics_type = "none"
@@ -22,17 +21,16 @@ def provider_libvirt(config)
     virt.memory = ENV.fetch('TFB_KVM_MEM', 3022)
     virt.cpus = ENV.fetch('TFB_KVM_CPU', 2)
 
-    override.vm.synced_folder "../../toolset", "/home/vagrant/FrameworkBenchmarks/toolset", type: "nfs"
-    override.vm.synced_folder "../../frameworks", "/home/vagrant/FrameworkBenchmarks/frameworks", type: "nfs"
-    override.vm.synced_folder "../../results", "/home/vagrant/FrameworkBenchmarks/results", type: "nfs", create: true
+    override.vm.synced_folder "../../toolset", "/home/vagrant/FrameworkBenchmarks/toolset", type: "nfs", nfs_udp: false
+    override.vm.synced_folder "../../frameworks", "/home/vagrant/FrameworkBenchmarks/frameworks", type: "nfs", nfs_udp: false
+    override.vm.synced_folder "../../results", "/home/vagrant/FrameworkBenchmarks/results", type: "nfs", nfs_udp: false, create: true
   end
 end
 
 def provider_virtualbox(config)
   config.vm.provider :virtualbox do |vb, override|
-    vb.name = "TechEmpower Framework Benchmarks"
     override.vm.hostname = "TFB-all"
-    override.vm.box = "ubuntu/trusty64"
+    override.vm.box = "ubuntu/xenial64"
 
     if ENV.fetch('TFB_SHOW_VM', false)
       vb.gui = true

frameworks/C++/cpoll_cppsp/Makefile

@@ -3,7 +3,7 @@ all: cppsp_0.2.3
 clean:
 	rm -f www/*.so www/*.txt
 	rm -f www/forcedynamic.cppsm.*
-	$(MAKE) -C $(IROOT)/cppsp_0.2.3 clean
+	$(MAKE) -C /installs/cppsp_0.2.3 clean
 
 cppsp_0.2.3:
-	$(MAKE) -C $(IROOT)/cppsp_0.2.3 all
+	$(MAKE) -C /installs/cppsp_0.2.3 all

frameworks/C++/cpoll_cppsp/benchmark_config.json

@@ -2,7 +2,6 @@
   "framework": "cpoll_cppsp",
   "tests": [{
     "default": {
-      "setup_file": "setup",
       "json_url": "/json",
       "plaintext_url": "/plaintext",
       "port": 16969,
@@ -22,7 +21,6 @@
       "versus": "cpoll_cppsp"
     },
     "raw": {
-      "setup_file": "setup_mysql",
       "db_url": "/db",
       "query_url": "/db?queries=",
       "fortune_url": "/fortune",
@@ -44,7 +42,6 @@
       "versus": "cpoll_cppsp"
     },
     "postgres-raw": {
-      "setup_file": "setup_postgresql",
       "db_url": "/db_pg_async",
       "query_url": "/db_pg_async?queries=", 
       "port": 16969,
@@ -64,7 +61,6 @@
       "versus": "cpoll_cppsp"
     },
     "postgres-raw-threadpool": {
-      "setup_file": "setup_postgresql",
       "db_url": "/db_pg_threadpool",
       "query_url": "/db_pg_threadpool?queries=", 
       "port": 16969,

frameworks/C++/cpoll_cppsp/cpoll_cppsp-base.dockerfile

@@ -0,0 +1,24 @@
+FROM techempower/gcc-4.8:0.1
+
+WORKDIR /installs
+
+ENV VERSION=0.2.3
+ENV CPPSP_HOME=/installs/cppsp_$VERSION
+
+RUN wget -q http://downloads.sourceforge.net/project/cpollcppsp/CPPSP%200.2%20%28testing%29/cppsp_$VERSION.tar.xz
+RUN tar xf cppsp_$VERSION.tar.xz
+
+RUN mv cppsp_rel$VERSION/ $CPPSP_HOME
+
+RUN sed -i 's|CXX := .*|CXX := g++-4.8|g' $CPPSP_HOME/makefile
+RUN sed -i 's|-Wall|-w|g' $CPPSP_HOME/makefile
+
+RUN apt install -yqq postgresql-server-dev-9.5
+ENV CPLUS_INCLUDE_PATH=/usr/include/postgresql:/usr/include/postgresql/9.5/server:${CPLUS_INCLUDE_PATH}
+
+ADD ./ /cpoll_cppsp
+WORKDIR /cpoll_cppsp
+
+RUN make clean && make
+
+WORKDIR $CPPSP_HOME

frameworks/C++/cpoll_cppsp/cpoll_cppsp-postgres-raw-threadpool.dockerfile

@@ -0,0 +1,3 @@
+FROM techempower/cpoll_cppsp-base:0.1
+
+CMD ./run_application /cpoll_cppsp/www -g g++-4.8 -m /forcedynamic.cppsm

frameworks/C++/cpoll_cppsp/cpoll_cppsp-postgres-raw.dockerfile

@@ -0,0 +1,3 @@
+FROM techempower/cpoll_cppsp-base:0.1
+
+CMD ./run_application /cpoll_cppsp/www -g g++-4.8 -m /forcedynamic.cppsm

frameworks/C++/cpoll_cppsp/cpoll_cppsp-raw.dockerfile

@@ -0,0 +1,3 @@
+FROM techempower/cpoll_cppsp-base:0.1
+
+CMD ./run_application /cpoll_cppsp/www -g g++-4.8 -m /forcedynamic.cppsm

frameworks/C++/cpoll_cppsp/cpoll_cppsp.dockerfile

@@ -0,0 +1,3 @@
+FROM techempower/cpoll_cppsp-base:0.1
+
+CMD ./run_application /cpoll_cppsp/www -g g++-4.8 -m /forcedynamic.cppsm

frameworks/C++/cpoll_cppsp/www/connectioninfo.H

@@ -1,4 +1,4 @@
-#define BENCHMARK_DB_HOST "TFB-database"
+#define BENCHMARK_DB_HOST "tfb-database"
 #define MYSQL_MAX_CONNECTIONS 3000
 
 #include <stdexcept>