Skip to content

Commit

Permalink
docker: fix docker image to suit the memory storage
Browse files Browse the repository at this point in the history 
Currently the datasets created in the init process are stored on disk,
and they are inaccessible if the “real” server is started with storage in memory.

This is because for the container initialization, the current logic is:
1. start_init_server: start the single-node server in the background, with
default on-disk storage setting;
2. run_init_queries: create default db and user, and run the init scripts
saved in
docker-entrypoint-initdb.d;
3. *stop the init server and restart the server* with given parameters in foreground.

In this commit we start and keep running the init server with the given
parameters, and eliminate the “stop and restart” step. Also, bring the server
process from background to foreground with fg once all init queries are finished.

Fixes cockroachdb#80005.

Release note (docker): refactor the initialization process of the docker image
to accomodate the use case with memory storage.
  • Loading branch information
@ZhouXing19
ZhouXing19 committed Apr 26, 2022
1 parent b7681be commit 82d1642
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 83 deletions.
89 changes: 23 additions & 66 deletions build/deploy/cockroach.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
set -eu

# set -e: If the command returns a non-zero exit status, exit the shell.
# set -u: errors if an variable is referenced before being set
# set -m: enable job control.
set -eum

cockroach_entrypoint="/cockroach/cockroach"

Expand All @@ -19,6 +23,7 @@ listen_addr=
advertise_addr=
default_listen_addr_host="127.0.0.1"
default_port="26257"
default_log_dir="./cockroach-data/logs"

advertise_addr_host=$default_listen_addr_host

Expand Down Expand Up @@ -126,12 +131,13 @@ setup_env() {
start_init_node() {
echo "starting node for the initialization process. This could take a couple seconds..."
rm -f server_fifo; mkfifo server_fifo
local start_node_query=( $cockroach_entrypoint start-single-node \
--background
local start_node_query=( exec $cockroach_entrypoint start-single-node \
--listening-url-file=server_fifo \
--pid-file=server_pid \
--advertise-addr="$advertise_addr" \
--certs-dir="$certs_dir" )
--certs-dir="$certs_dir" \
--log="file-defaults: {dir: $default_log_dir}" \
"$@" )

# Start the node and run in the background.
"${start_node_query[@]}" &
Expand Down Expand Up @@ -168,6 +174,7 @@ setup_db() {
# usage: process_init_files [file [file [...]]]
# e.g. process_init_files /your_folder/*
process_init_files() {
echo "start running init files from /docker-entrypoint-initdb.d"
for f in "$@"; do
case "$f" in
*.sh)
Expand All @@ -187,20 +194,7 @@ process_init_files() {
esac
echo
done
}

# stop_init_node is to stop the single node for the initialization.
stop_init_node() {
kill $(cat server_pid)
local timeout=100
local time_counter=0
until [[ $time_counter -eq $timeout ]] || kill -0 $(cat server_pid); do
echo >&2 "finishing cockroach init process"
sleep 2
time_counter=$((time_counter+1))
done
check_if_server_fully_stopped
echo >&2 "cockroach init process finished, restart the server now"
echo "end running init files from /docker-entrypoint-initdb.d"
}

# run_sql_query is a helper function to run sql queries.
Expand Down Expand Up @@ -257,64 +251,27 @@ create_default_user() {
fi
}

# check_if_server_fully_stopped is to wait until the init server is fully
# stopped or timeout.
check_if_server_fully_stopped() {
local log_path=./cockroach-data/logs/cockroach.log
local timeout=20
local time_counter=0

until [[ ( $time_counter -eq $timeout ) || ( -f $log_path ) ]]; do
echo >&2 "$log_path doesn't exist, waiting ..."
sleep 2
time_counter=$((time_counter+1))
done

# If timeout, exit the program.
if [[ $time_counter -ge $timeout ]]; then
echo >&2 "error: timeout for finding log file"
exit 1
fi

time_counter=0
echo >&2 "waiting for the init server to be fully stopped..."

# Wait until either the last line of the log contains "server drained and
# shutdown completed" or timeout.
until [[ ( $time_counter -eq $timeout ) || \
( "$(tail -1 $log_path)" == *"server drained and shutdown completed"* ) ]];
do
sleep 2
time_counter=$((time_counter+1))
done

# If timeout, exit the program.
if [[ $time_counter -ge $timeout ]]; then
echo >&2 "error: timeout for stopping the init server"
exit 1
fi
echo >&2 "init server fully stopped"
}

# run_single_node process the command if it contains `start-single-node` argument.
run_single_node() {
# If /cockroach-data is empty, run the initialization steps.
if [[ $(ls -A cockroach-data | wc -l) = 0 ]]; then
setup_certs_dir
setup_env
# Start the init server.
# Start the server.
start_init_node "$@"
setup_db "$@"
process_init_files /docker-entrypoint-initdb.d/*
# Stop the init server.
stop_init_node
touch init_success
# Bring the background server process to the foreground, otherwise the
# docker container will automatically exit here.
echo "init_finished" > ./init_success
fg %1
else
exec $cockroach_entrypoint start-single-node \
--certs-dir="$certs_dir" \
--advertise-addr=$advertise_addr \
"$@"
fi
# Start the real server.
exec $cockroach_entrypoint start-single-node \
--certs-dir="$certs_dir" \
--advertise-addr=$advertise_addr \
"$@"

}

_main() {
Expand Down
58 changes: 41 additions & 17 deletions pkg/testutils/docker/single_node_docker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,33 @@ func TestSingleNodeDocker(t *testing.T) {
{"SELECT * FROM bello", "id,name\n1,a\n2,b\n3,c"},
},
},
{
testName: "single-node-insecure-mem-mode",
containerName: "roach3",
runContainerArgs: runContainerArgs{
envSetting: []string{
"COCKROACH_DATABASE=mydb",
},
volSetting: []string{
fmt.Sprintf("%s/testdata/single-node-test/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d", pwd),
fmt.Sprintf("%s/docker-fsnotify-bin:/cockroach/docker-fsnotify", fsnotifyPath),
},
cmd: []string{"start-single-node", "--insecure", "--store=type=mem,size=0.25"},
},
sqlOpts: []string{
"--format=csv",
"--insecure",
"--database=mydb",
},
sqlQueries: []sqlQuery{
{"SELECT current_user", "current_user\nroot"},
{"SELECT current_database()", "current_database\nmydb"},
{"CREATE TABLE hello (X INT)", "CREATE TABLE"},
{"INSERT INTO hello VALUES (1), (2), (3)", "INSERT 3"},
{"SELECT * FROM hello", "x\n1\n2\n3"},
{"SELECT * FROM bello", "id,name\n1,a\n2,b\n3,c"},
},
},
}

cl, err := client.NewClientWithOpts(client.FromEnv)
Expand Down Expand Up @@ -187,10 +214,10 @@ func TestSingleNodeDocker(t *testing.T) {

if err := contextutil.RunWithTimeout(
ctx,
"wait for the server to fully start up",
serverStartTimeout,
"wait for the server to finish the initialization",
waitInitTimeout,
func(ctx context.Context) error {
return dn.waitServerStarts(ctx)
return dn.waitInitFinishes(ctx)
},
); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -253,8 +280,8 @@ func TestSingleNodeDocker(t *testing.T) {
const (
imageName = "cockroachdb/cockroach-ci:latest"
defaultTimeout = 10 * time.Second
serverStartTimeout = 80 * time.Second
listenURLFile = "demoFile"
waitInitTimeout = 80 * time.Second
initSuccessFile = "init_success"
cockroachEntrypoint = "./cockroach"
hostPort = "8080"
cockroachPort = "26257"
Expand Down Expand Up @@ -328,7 +355,7 @@ func (dn *dockerNode) startContainer(
Image: imageName,
Env: envSetting,
ExposedPorts: nat.PortSet{hostPort: struct{}{}, cockroachPort: struct{}{}},
Cmd: append(cmd, fmt.Sprintf("--listening-url-file=%s", listenURLFile)),
Cmd: cmd,
}

hostConfig := container.HostConfig{
Expand Down Expand Up @@ -484,13 +511,10 @@ func (dn *dockerNode) execCommand(
return &res, nil
}

// waitServerStarts waits till the server truly starts or timeout, whichever
// earlier. It keeps listening to the listenURLFile till it is closed and
// written. This is because in #70238, the improved init process for single-node
// server is to start the server, run the init process, and then restart the
// server. We mark it as fully started until the server is successfully
// restarted, and hence write the url to listenURLFile.
func (dn *dockerNode) waitServerStarts(ctx context.Context) error {
// waitInitFinishes waits till the server finishes all init steps or timeout,
// whichever earlier. It keeps listening to the initSuccessFile till it is closed and
// written.
func (dn *dockerNode) waitInitFinishes(ctx context.Context) error {
var res *execResult
var err error

Expand All @@ -499,17 +523,17 @@ func (dn *dockerNode) waitServerStarts(ctx context.Context) error {
res, err = dn.execCommand(ctx, []string{
"./docker-fsnotify",
"/cockroach",
listenURLFile,
strconv.Itoa(int(serverStartTimeout.Seconds())),
initSuccessFile,
strconv.Itoa(int(waitInitTimeout.Seconds())),
}, "/cockroach")
if err != nil {
return errors.Wrapf(err, "cannot run fsnotify to listen to %s:\nres:%#v", listenURLFile, res)
return errors.Wrapf(err, "cannot run fsnotify to listen to %s:\nres:%#v\n", initSuccessFile, res)
}

if strings.Contains(res.stdOut, "finished\r\n") {
return nil
}
return errors.Wrap(errors.Newf("%s", res), "error in waiting the server to start")
return errors.Wrap(errors.Newf("%#v", res), "error waiting the initialization to finish")
}

// execSQLQuery executes the sql query and returns the server's output and
Expand Down

0 comments on commit 82d1642

Please sign in to comment.