Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

release-22.1: docker: fix docker image to suit storage in memory #80355

Merged
merged 2 commits into from
Apr 22, 2022
Merged

release-22.1: docker: fix docker image to suit storage in memory #80355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c6634f6
docker: fix docker image to suit the memory storage
ZhouXing19 Apr 20, 2022
7b2bce8
list files
ZhouXing19 Apr 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 23 additions & 66 deletions build/deploy/cockroach.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
set -eu

# set -e: If the command returns a non-zero exit status, exit the shell.
# set -u: errors if an variable is referenced before being set
# set -m: enable job control.
set -eum

cockroach_entrypoint="/cockroach/cockroach"

Expand All @@ -19,6 +23,7 @@ listen_addr=
advertise_addr=
default_listen_addr_host="127.0.0.1"
default_port="26257"
default_log_dir="./cockroach-data/logs"

advertise_addr_host=$default_listen_addr_host

Expand Down Expand Up @@ -126,12 +131,13 @@ setup_env() {
start_init_node() {
echo "starting node for the initialization process. This could take a couple seconds..."
rm -f server_fifo; mkfifo server_fifo
local start_node_query=( $cockroach_entrypoint start-single-node \
--background
local start_node_query=( exec $cockroach_entrypoint start-single-node \
--listening-url-file=server_fifo \
--pid-file=server_pid \
--advertise-addr="$advertise_addr" \
--certs-dir="$certs_dir" )
--certs-dir="$certs_dir" \
--log="file-defaults: {dir: $default_log_dir}" \
"$@" )

# Start the node and run in the background.
"${start_node_query[@]}" &
Expand Down Expand Up @@ -168,6 +174,7 @@ setup_db() {
# usage: process_init_files [file [file [...]]]
# e.g. process_init_files /your_folder/*
process_init_files() {
echo "start running init files from /docker-entrypoint-initdb.d"
for f in "$@"; do
case "$f" in
*.sh)
Expand All @@ -187,20 +194,7 @@ process_init_files() {
esac
echo
done
}

# stop_init_node is to stop the single node for the initialization.
stop_init_node() {
kill $(cat server_pid)
local timeout=100
local time_counter=0
until [[ $time_counter -eq $timeout ]] || kill -0 $(cat server_pid); do
echo >&2 "finishing cockroach init process"
sleep 2
time_counter=$((time_counter+1))
done
check_if_server_fully_stopped
echo >&2 "cockroach init process finished, restart the server now"
echo "end running init files from /docker-entrypoint-initdb.d"
}

# run_sql_query is a helper function to run sql queries.
Expand Down Expand Up @@ -257,64 +251,27 @@ create_default_user() {
fi
}

# check_if_server_fully_stopped is to wait until the init server is fully
# stopped or timeout.
check_if_server_fully_stopped() {
local log_path=./cockroach-data/logs/cockroach.log
local timeout=20
local time_counter=0

until [[ ( $time_counter -eq $timeout ) || ( -f $log_path ) ]]; do
echo >&2 "$log_path doesn't exist, waiting ..."
sleep 2
time_counter=$((time_counter+1))
done

# If timeout, exit the program.
if [[ $time_counter -ge $timeout ]]; then
echo >&2 "error: timeout for finding log file"
exit 1
fi

time_counter=0
echo >&2 "waiting for the init server to be fully stopped..."

# Wait until either the last line of the log contains "server drained and
# shutdown completed" or timeout.
until [[ ( $time_counter -eq $timeout ) || \
( "$(tail -1 $log_path)" == *"server drained and shutdown completed"* ) ]];
do
sleep 2
time_counter=$((time_counter+1))
done

# If timeout, exit the program.
if [[ $time_counter -ge $timeout ]]; then
echo >&2 "error: timeout for stopping the init server"
exit 1
fi
echo >&2 "init server fully stopped"
}

# run_single_node process the command if it contains `start-single-node` argument.
run_single_node() {
# If /cockroach-data is empty, run the initialization steps.
if [[ $(ls -A cockroach-data | wc -l) = 0 ]]; then
setup_certs_dir
setup_env
# Start the init server.
# Start the server.
start_init_node "$@"
setup_db "$@"
process_init_files /docker-entrypoint-initdb.d/*
# Stop the init server.
stop_init_node
touch init_success
# Bring the background server process to the foreground, otherwise the
# docker container will automatically exit here.
echo "init_finished" > ./init_success
fg %1
else
exec $cockroach_entrypoint start-single-node \
--certs-dir="$certs_dir" \
--advertise-addr=$advertise_addr \
"$@"
fi
# Start the real server.
exec $cockroach_entrypoint start-single-node \
--certs-dir="$certs_dir" \
--advertise-addr=$advertise_addr \
"$@"

}

_main() {
Expand Down
58 changes: 41 additions & 17 deletions pkg/testutils/docker/single_node_docker_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,33 @@ func TestSingleNodeDocker(t *testing.T) {
{"SELECT * FROM bello", "id,name\n1,a\n2,b\n3,c"},
},
},
{
testName: "single-node-insecure-mem-mode",
containerName: "roach3",
runContainerArgs: runContainerArgs{
envSetting: []string{
"COCKROACH_DATABASE=mydb",
},
volSetting: []string{
fmt.Sprintf("%s/testdata/single-node-test/docker-entrypoint-initdb.d/:/docker-entrypoint-initdb.d", pwd),
fmt.Sprintf("%s/docker-fsnotify-bin:/cockroach/docker-fsnotify", fsnotifyPath),
},
cmd: []string{"start-single-node", "--insecure", "--store=type=mem,size=0.25"},
},
sqlOpts: []string{
"--format=csv",
"--insecure",
"--database=mydb",
},
sqlQueries: []sqlQuery{
{"SELECT current_user", "current_user\nroot"},
{"SELECT current_database()", "current_database\nmydb"},
{"CREATE TABLE hello (X INT)", "CREATE TABLE"},
{"INSERT INTO hello VALUES (1), (2), (3)", "INSERT 3"},
{"SELECT * FROM hello", "x\n1\n2\n3"},
{"SELECT * FROM bello", "id,name\n1,a\n2,b\n3,c"},
},
},
}

cl, err := client.NewClientWithOpts(client.FromEnv)
Expand Down Expand Up @@ -187,10 +214,10 @@ func TestSingleNodeDocker(t *testing.T) {

if err := contextutil.RunWithTimeout(
ctx,
"wait for the server to fully start up",
serverStartTimeout,
"wait for the server to finish the initialization",
waitInitTimeout,
func(ctx context.Context) error {
return dn.waitServerStarts(ctx)
return dn.waitInitFinishes(ctx)
},
); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -253,8 +280,8 @@ func TestSingleNodeDocker(t *testing.T) {
const (
imageName = "cockroachdb/cockroach-ci:latest"
defaultTimeout = 10 * time.Second
serverStartTimeout = 80 * time.Second
listenURLFile = "demoFile"
waitInitTimeout = 80 * time.Second
initSuccessFile = "init_success"
cockroachEntrypoint = "./cockroach"
hostPort = "8080"
cockroachPort = "26257"
Expand Down Expand Up @@ -328,7 +355,7 @@ func (dn *dockerNode) startContainer(
Image: imageName,
Env: envSetting,
ExposedPorts: nat.PortSet{hostPort: struct{}{}, cockroachPort: struct{}{}},
Cmd: append(cmd, fmt.Sprintf("--listening-url-file=%s", listenURLFile)),
Cmd: cmd,
}

hostConfig := container.HostConfig{
Expand Down Expand Up @@ -484,13 +511,10 @@ func (dn *dockerNode) execCommand(
return &res, nil
}

// waitServerStarts waits till the server truly starts or timeout, whichever
// earlier. It keeps listening to the listenURLFile till it is closed and
// written. This is because in #70238, the improved init process for single-node
// server is to start the server, run the init process, and then restart the
// server. We mark it as fully started until the server is successfully
// restarted, and hence write the url to listenURLFile.
func (dn *dockerNode) waitServerStarts(ctx context.Context) error {
// waitInitFinishes waits till the server finishes all init steps or timeout,
// whichever earlier. It keeps listening to the initSuccessFile till it is closed and
// written.
func (dn *dockerNode) waitInitFinishes(ctx context.Context) error {
var res *execResult
var err error

Expand All @@ -499,17 +523,17 @@ func (dn *dockerNode) waitServerStarts(ctx context.Context) error {
res, err = dn.execCommand(ctx, []string{
"./docker-fsnotify",
"/cockroach",
listenURLFile,
strconv.Itoa(int(serverStartTimeout.Seconds())),
initSuccessFile,
strconv.Itoa(int(waitInitTimeout.Seconds())),
}, "/cockroach")
if err != nil {
return errors.Wrapf(err, "cannot run fsnotify to listen to %s:\nres:%#v", listenURLFile, res)
return errors.Wrapf(err, "cannot run fsnotify to listen to %s:\nres:%#v\n", initSuccessFile, res)
}

if strings.Contains(res.stdOut, "finished\r\n") {
return nil
}
return errors.Wrap(errors.Newf("%s", res), "error in waiting the server to start")
return errors.Wrap(errors.Newf("%#v", res), "error waiting the initialization to finish")
}

// execSQLQuery executes the sql query and returns the server's output and
Expand Down