Update images used in benchmarking

`glassfish` image is officially deprecated on DockerHub and some images used in our benchmarking are old. This commit replaces and upgrades these images. - `glassfish:4.1-jdk8` -> `tomcat:10.0.0-jdk15-openjdk-buster` - `glassfish` is deprecated. - `rethinkdb:2.3.6` -> `postgres:13.1` - The latest `rethinkdb` is too small. - `python:3.7` -> `python:3.9` - `gcc:9.2.0` -> `gcc:10.2.0` Signed-off-by: Kohei Tokunaga <[email protected]>
containerd · Jan 26, 2021 · b07c95e · b07c95e
1 parent 9769fd4
commit b07c95e
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 12 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -13,7 +13,7 @@ jobs:
       BENCHMARK_RESULT_DIR: ${{ github.workspace }}/benchmark/
       BENCHMARK_REGISTRY: ghcr.io
       BENCHMARK_USER: stargz-containers
-      BENCHMARK_TARGETS: python:3.7 gcc:9.2.0 rethinkdb:2.3.6 glassfish:4.1-jdk8
+      BENCHMARK_TARGETS: python:3.9 gcc:10.2.0 postgres:13.1 tomcat:10.0.0-jdk15-openjdk-buster
       BENCHMARK_SAMPLES_NUM: 5
       BENCHMARK_PERCENTILE: 95
       BENCHMARK_PERCENTILES_GRANULARITY: 25

diff --git a/docs/pre-converted-images.md b/docs/pre-converted-images.md
@@ -25,10 +25,8 @@ Additionally, image names listed in `Image Name` contain the following suffixes
 |`ghcr.io/stargz-containers/drupal:8.7.6-esgz`|Code execution until up and ready message (`apache2 -D FOREGROUND`) is printed|
 |`ghcr.io/stargz-containers/fedora:30-org`|Executing `echo hello` on the shell|
 |`ghcr.io/stargz-containers/fedora:30-esgz`|Executing `echo hello` on the shell|
-|`ghcr.io/stargz-containers/gcc:9.2.0-org`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/gcc:9.2.0-esgz`|Compiling and executing a program which prints `hello`|
-|`ghcr.io/stargz-containers/glassfish:4.1-jdk8-org`|Code execution until up and ready message (`Running GlassFish`) is printed|
-|`ghcr.io/stargz-containers/glassfish:4.1-jdk8-esgz`|Code execution until up and ready message (`Running GlassFish`) is printed|
+|`ghcr.io/stargz-containers/gcc:10.2.0-org`|Compiling and executing a program which prints `hello`|
+|`ghcr.io/stargz-containers/gcc:10.2.0-esgz`|Compiling and executing a program which prints `hello`|
 |`ghcr.io/stargz-containers/golang:1.12.9-org`|Compiling and executing a program which prints `hello`|
 |`ghcr.io/stargz-containers/golang:1.12.9-esgz`|Compiling and executing a program which prints `hello`|
 |`ghcr.io/stargz-containers/jenkins:2.60.3-org`|Code execution until up and ready message (`Jenkins is fully up and running`) is printed|
@@ -43,11 +41,15 @@ Additionally, image names listed in `Image Name` contain the following suffixes
 |`ghcr.io/stargz-containers/php:7.3.8-esgz`|Printing `hello`|
 |`ghcr.io/stargz-containers/pypy:3.5-org`|Printing `hello`|
 |`ghcr.io/stargz-containers/pypy:3.5-esgz`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.7-org`|Printing `hello`|
-|`ghcr.io/stargz-containers/python:3.7-esgz`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.9-org`|Printing `hello`|
+|`ghcr.io/stargz-containers/python:3.9-esgz`|Printing `hello`|
 |`ghcr.io/stargz-containers/r-base:3.6.1-org`|Printing `hello`|
 |`ghcr.io/stargz-containers/r-base:3.6.1-esgz`|Printing `hello`|
 |`ghcr.io/stargz-containers/redis:5.0.5-org`|Code execution until up and ready message (`Ready to accept connections`) is printed|
 |`ghcr.io/stargz-containers/redis:5.0.5-esgz`|Code execution until up and ready message (`Ready to accept connections`) is printed|
 |`ghcr.io/stargz-containers/rethinkdb:2.3.6-org`|Code execution until up and ready message (`Server ready`) is printed|
 |`ghcr.io/stargz-containers/rethinkdb:2.3.6-esgz`|Code execution until up and ready message (`Server ready`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.0.0-jdk15-openjdk-buster-org`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/tomcat:10.0.0-jdk15-openjdk-buster-esgz`|Code execution until up and ready message (`Server startup`) is printed|
+|`ghcr.io/stargz-containers/postgres:13.1-org`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
+|`ghcr.io/stargz-containers/postgres:13.1-esgz`|Code execution until up and ready message (`database system is ready to accept connections`) is printed|
diff --git a/script/benchmark/hello-bench/src/hello.py b/script/benchmark/hello-bench/src/hello.py
@@ -93,10 +93,13 @@ class BenchRunner:
                     'drupal:8.7.6': RunArgs(waitline='apache2 -D FOREGROUND'),
                     'jenkins:2.60.3': RunArgs(waitline='Jenkins is fully up and running'),
                     'redis:5.0.5': RunArgs(waitline='Ready to accept connections'),
+                    'tomcat:10.0.0-jdk15-openjdk-buster': RunArgs(waitline='Server startup'),
+                    'postgres:13.1': RunArgs(waitline='database system is ready to accept connections',
+                                             env={'POSTGRES_PASSWORD': 'abc'}),
     }
 
     CMD_STDIN = {'php:7.3.8':  RunArgs(stdin='php -r "echo \\\"hello\\n\\\";"; exit\n'),
-                 'gcc:9.2.0': RunArgs(stdin='cd /src; gcc main.c; ./a.out; exit\n',
+                 'gcc:10.2.0': RunArgs(stdin='cd /src; gcc main.c; ./a.out; exit\n',
                                 mount=[('gcc', '/src')]),
                  'golang:1.12.9': RunArgs(stdin='cd /go/src; go run main.go; exit\n',
                                    mount=[('go', '/go/src')]),
@@ -105,7 +108,7 @@ class BenchRunner:
     }
 
     CMD_ARG = {'perl:5.30': RunArgs(arg='perl -e \'print("hello\\n")\''),
-               'python:3.7': RunArgs(arg='python -c \'print("hello")\''),
+               'python:3.9': RunArgs(arg='python -c \'print("hello")\''),
                'pypy:3.5': RunArgs(arg='pypy3 -c \'print("hello")\''),
                'node:13.13.0': RunArgs(arg='node -e \'console.log("hello")\''),
     }
@@ -115,19 +118,20 @@ class BenchRunner:
                 [Bench('alpine:3.10.2', 'distro'),
                  Bench('fedora:30', 'distro'),
                  Bench('rethinkdb:2.3.6', 'database'),
+                 Bench('postgres:13.1', 'database'),
                  Bench('redis:5.0.5', 'database'),
-                 Bench('python:3.7', 'language'),
+                 Bench('python:3.9', 'language'),
                  Bench('golang:1.12.9', 'language'),
-                 Bench('gcc:9.2.0', 'language'),
+                 Bench('gcc:10.2.0', 'language'),
                  Bench('jruby:9.2.8.0', 'language'),
                  Bench('perl:5.30', 'language'),
                  Bench('php:7.3.8', 'language'),
                  Bench('pypy:3.5', 'language'),
                  Bench('r-base:3.6.1', 'language'),
-                 Bench('glassfish:4.1-jdk8', 'web-server'),
                  Bench('drupal:8.7.6'),
                  Bench('jenkins:2.60.3'),
                  Bench('node:13.13.0'),
+                 Bench('tomcat:10.0.0-jdk15-openjdk-buster', 'web-server'),
              ]])
 
     def __init__(self, repository='docker.io/library', mode=LEGACY_MODE, optimizer=DEFAULT_OPTIMIZER):