Skip to content

Commit

Permalink
Persist metadata
Browse files Browse the repository at this point in the history
FDQN mode is required to use hostnames rather than IP addresses as node
IDs since docker changes the IP addresses when recreating containers.
  • Loading branch information
MrCreosote committed Jun 14, 2024
1 parent 3130037 commit 807f1b4
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 17 deletions.
10 changes: 5 additions & 5 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ services:
container_name: starrocks-fe
user: root
environment:
- STARROCKS_HOME=/starrocks/fe1
- STARROCKS_DATA=/starrocks_mount/fe1
- HTTP_PORT=8031
- THRIFT_RPC_PORT=9021
- SQL_QUERY_PORT=9031
Expand All @@ -63,7 +63,7 @@ services:
- 9021:9021
- 9031:9031
volumes:
- ./cdr/cdm/starrocks:/starrocks
- ./cdr/cdm/starrocks:/starrocks_mount
healthcheck:
test: 'mysql -u root -h starrocks-fe -P 9031 -e "show frontends\G" |grep "Alive: true"'
interval: 10s
Expand All @@ -73,14 +73,14 @@ services:
- minio

# NOTE: To add the CN to the system, you need to manually run
# ALTER SYSTEM ADD COMPUTE NODE "starrocks-cn:9051"
# ALTER SYSTEM ADD COMPUTE NODE "starrocks-cn:9051";
# (or whatever port / host name you choose) on the FE node
starrocks-cn:
build:
context: .
dockerfile: ./DockerFile_ComputeNode
environment:
- STARROCKS_HOME=/starrocks/cn1
- STARROCKS_DATA=/starrocks_mount/cn1
- SLEEP_FOR=15s
- THRIFT_RPC_PORT=9061
- HTTP_PORT=8041
Expand All @@ -93,7 +93,7 @@ services:
container_name: starrocks-cn
user: root
volumes:
- ./cdr/cdm/starrocks:/starrocks
- ./cdr/cdm/starrocks:/starrocks_mount
depends_on:
- starrocks-fe
- minio
7 changes: 2 additions & 5 deletions scripts/cn-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
#!/usr/bin/env bash

# Note that the STARROCKS_HOME variable should also be set to
# determine where to save metadata, logs, etc.

mkdir -p $STARROCKS_HOME
mkdir -p $STARROCKS_DATA

export THRIFT_RPC_PORT=${THRIFT_RPC_PORT:-9060}
export HTTP_PORT=${HTTP_PORT:-8040}
export HEARTBEAT_PORT=${HEARTBEAT_PORT:-9050}
export BRPC_PORT=${BRPC_PORT:-8060}
export STARLET_PORT=${STARLET_PORT:-9070}

echo "STARROCKS_HOME=${STARROCKS_HOME}"
echo "STARROCKS_DATA=${STARROCKS_DATA}"
echo "THRIFT_RPC_PORT=${THRIFT_RPC_PORT}"
echo "HTTP_PORT=${HTTP_PORT}"
echo "HEARTBEAT_PORT=${HEARTBEAT_PORT}"
Expand Down
21 changes: 21 additions & 0 deletions scripts/cn-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,29 @@ ulimit -n 65535

TF=/opt/starrocks/cn/conf/cn.conf

# Make ports configurable
sed -i "s#^be_port =.*#be_port = \${THRIFT_RPC_PORT}#" $TF
sed -i "s#^be_http_port =.*#be_http_port = \${HTTP_PORT}#" $TF
sed -i "s#^heartbeat_service_port =.*#heartbeat_service_port = \${HEARTBEAT_PORT}#" $TF
sed -i "s#^brpc_port =.*#brpc_port = \${BRPC_PORT}#" $TF
sed -i "s#^starlet_port =.*#starlet_port = \${STARLET_PORT}#" $TF

# Set up data config
echo 'block_cache_disk_path=${STARROCKS_DATA}/block_cache/' >> $TF
echo 'block_cache_meta_path=${STARROCKS_DATA}/block_cache/' >> $TF
echo 'datacache_disk_path=${STARROCKS_DATA}/datacache/' >> $TF
echo 'datacache_meta_path=${STARROCKS_DATA}/datacache/' >> $TF
echo 'user_function_dir=${STARROCKS_DATA}/lib/udf' >> $TF
echo 'local_library_dir=${STARROCKS_DATA}/lib/udf-runtime' >> $TF
echo 'pprof_profile_dir=${STARROCKS_DATA}/log' >> $TF
echo 'query_scratch_dirs=${STARROCKS_DATA}' >> $TF
echo 'small_file_dir=${STARROCKS_DATA}/lib/small_file/' >> $TF
echo 'spill_local_storage_dir=${STARROCKS_DATA}/spill' >> $TF
echo 'starlet_cache_dir=${STARROCKS_DATA}/storage/starlet_cache' >> $TF
echo 'storage_root_path=${STARROCKS_DATA}/storage' >> $TF
echo 'sys_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'sys_minidump_dir=${STARROCKS_DATA}' >> $TF

# I frankly have no idea what these are or whether they need to be persisted
echo 'pull_load_task_dir=${STARROCKS_DATA}/var/pull_load' >> $TF
echo 'query_debug_trace_dir=${STARROCKS_DATA}/query_debug_trace' >> $TF
11 changes: 4 additions & 7 deletions scripts/fe-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
#!/bin/sh

# Note that the STARROCKS_HOME variable should also be set to
# determine where to save metadata, logs, etc.

mkdir -p $STARROCKS_HOME
mkdir -p $STARROCKS_DATA/meta

# NOTE: All FE nodes in a cluster must have the same http port:
# https://docs.starrocks.io/docs/deployment/deploy_manually/
Expand All @@ -13,7 +10,7 @@ export THRIFT_RPC_PORT=${THRIFT_RPC_PORT:-9020}
export SQL_QUERY_PORT=${SQL_QUERY_PORT:-9030}
export EDIT_LOG_PORT=${EDIT_LOG_PORT:-9010}

echo "STARROCKS_HOME=${STARROCKS_HOME}"
echo "STARROCKS_DATA=${STARROCKS_DATA}"
echo "HTTP_PORT=${HTTP_PORT}"
echo "THRIFT_RPC_PORT=${THRIFT_RPC_PORT}"
echo "SQL_QUERY_PORT=${SQL_QUERY_PORT}"
Expand All @@ -23,7 +20,7 @@ echo "START_FOLLOWER_W_LEADER_EDIT_LOG_URL=${START_FOLLOWER_W_LEADER_EDIT_LOG_UR
# NOTE: this env var is only required the first time the follower is added to the cluster
# See https://docs.starrocks.io/docs/deployment/deploy_manually/
if [ -n "$START_FOLLOWER_W_LEADER_EDIT_LOG_URL" ]; then
/opt/starrocks/fe/bin/start_fe.sh --helper $START_FOLLOWER_W_LEADER_EDIT_LOG_URL;
/opt/starrocks/fe/bin/start_fe.sh --host_type FQDN --helper $START_FOLLOWER_W_LEADER_EDIT_LOG_URL;
else
/opt/starrocks/fe/bin/start_fe.sh;
/opt/starrocks/fe/bin/start_fe.sh --host_type FQDN;
fi
20 changes: 20 additions & 0 deletions scripts/fe-setup.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
#!/bin/sh

# TODO: image buids

TF=/opt/starrocks/fe/conf/fe.conf

# NOTE: All FE nodes in a cluster must have the same http port:
# https://docs.starrocks.io/docs/deployment/deploy_manually/

# Make ports configurable
sed -i "s#^http_port =.*#http_port = \${HTTP_PORT}#" $TF
sed -i "s#^rpc_port =.*#rpc_port = \${THRIFT_RPC_PORT}#" $TF
sed -i "s#^query_port =.*#query_port = \${SQL_QUERY_PORT}#" $TF
sed -i "s#^edit_log_port =.*#edit_log_port = \${EDIT_LOG_PORT}#" $TF

# Store log data in data dir
sed -i "s#^LOG_DIR = \${STARROCKS_HOME}/log$#LOG_DIR = \${STARROCKS_DATA}/log#" $TF

# Set up Minio
echo "# enable shared data, set storage type, set endpoint" >> $TF
echo "run_mode = shared_data" >> $TF
echo "cloud_native_storage_type = S3" >> $TF
Expand All @@ -28,3 +35,16 @@ echo "# Set this to false if you do not want default" >> $TF
echo "# storage created in the object storage using" >> $TF
echo "# the details provided above" >> $TF
echo "enable_load_volume_from_conf = true" >> $TF

# Set up data config
echo 'meta_dir=${STARROCKS_DATA}/meta' >> $TF
echo 'small_file_dir=${STARROCKS_DATA}/small_files' >> $TF
echo 'tmp_dir=${STARROCKS_DATA}/temp_dir' >> $TF
echo 'sys_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'big_query_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'profile_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'internal_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'dump_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'audit_log_dir=${STARROCKS_DATA}/log' >> $TF
echo 'spark_launcher_log_dir=${STARROCKS_DATA}/log/spark_launcher_log' >> $TF
echo 'iceberg_metadata_cache_disk_path=${STARROCKS_DATA}/caches/iceberg' >> $TF

0 comments on commit 807f1b4

Please sign in to comment.