Skip to content

Commit

Permalink
feat(FQDN): Replace rpc_address to host_prot for function & Add host_…
Browse files Browse the repository at this point in the history 
…port on thrift struct
  • Loading branch information
@GehaFearless
GehaFearless committed Feb 19, 2024
1 parent 9f41be7 commit 5ddb8a1
Show file tree
Hide file tree
Showing 182 changed files with 3,981 additions and 2,875 deletions.
72 changes: 38 additions & 34 deletions idl/bulk_load.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,16 @@ struct partition_bulk_load_state
// meta server -> replica server
struct bulk_load_request
{
1:dsn.gpid pid;
2:string app_name;
3:dsn.rpc_address primary_addr;
4:string remote_provider_name;
5:string cluster_name;
6:i64 ballot;
7:bulk_load_status meta_bulk_load_status;
8:bool query_bulk_load_metadata;
9:string remote_root_path;
1:dsn.gpid pid;
2:string app_name;
3:dsn.rpc_address primary_addr;
4:string remote_provider_name;
5:string cluster_name;
6:i64 ballot;
7:bulk_load_status meta_bulk_load_status;
8:bool query_bulk_load_metadata;
9:string remote_root_path;
10:optional dsn.host_port hp_primary;
}

struct bulk_load_response
Expand All @@ -109,28 +110,30 @@ struct bulk_load_response
// - ERR_FILE_OPERATION_FAILED: local file system error during bulk load downloading
// - ERR_FS_INTERNAL: remote file provider error during bulk load downloading
// - ERR_CORRUPTION: metadata corruption during bulk load downloading
1:dsn.error_code err;
2:dsn.gpid pid;
3:string app_name;
4:bulk_load_status primary_bulk_load_status;
5:map<dsn.rpc_address, partition_bulk_load_state> group_bulk_load_state;
6:optional bulk_load_metadata metadata;
7:optional i32 total_download_progress;
8:optional bool is_group_ingestion_finished;
9:optional bool is_group_bulk_load_context_cleaned_up;
10:optional bool is_group_bulk_load_paused;
1:dsn.error_code err;
2:dsn.gpid pid;
3:string app_name;
4:bulk_load_status primary_bulk_load_status;
5:map<dsn.rpc_address, partition_bulk_load_state> group_bulk_load_state;
6:optional bulk_load_metadata metadata;
7:optional i32 total_download_progress;
8:optional bool is_group_ingestion_finished;
9:optional bool is_group_bulk_load_context_cleaned_up;
10:optional bool is_group_bulk_load_paused;
11:optional map<dsn.host_port, partition_bulk_load_state> hp_group_bulk_load_state;
}

// primary -> secondary
struct group_bulk_load_request
{
1:string app_name;
2:dsn.rpc_address target_address;
3:metadata.replica_configuration config;
4:string provider_name;
5:string cluster_name;
6:bulk_load_status meta_bulk_load_status;
7:string remote_root_path;
1:string app_name;
2:dsn.rpc_address target;
3:metadata.replica_configuration config;
4:string provider_name;
5:string cluster_name;
6:bulk_load_status meta_bulk_load_status;
7:string remote_root_path;
8:optional dsn.host_port hp_target;
}

struct group_bulk_load_response
Expand Down Expand Up @@ -209,15 +212,16 @@ struct query_bulk_load_response
// - ERR_CORRUPTION: file not exist or damaged
// - ERR_INGESTION_FAILED: ingest failed
// - ERR_RETRY_EXHAUSTED: retry too many times
1:dsn.error_code err;
2:string app_name;
3:bulk_load_status app_status;
4:list<bulk_load_status> partitions_status;
5:i32 max_replica_count;
1:dsn.error_code err;
2:string app_name;
3:bulk_load_status app_status;
4:list<bulk_load_status> partitions_status;
5:i32 max_replica_count;
// detailed bulk load state for each replica
6:list<map<dsn.rpc_address, partition_bulk_load_state>> bulk_load_states;
7:optional string hint_msg;
8:optional bool is_bulk_loading;
6:list<map<dsn.rpc_address, partition_bulk_load_state>> bulk_load_states;
7:optional string hint_msg;
8:optional bool is_bulk_loading;
9:optional list<map<dsn.host_port, partition_bulk_load_state>> hp_bulk_load_states;
}

struct clear_bulk_load_state_request
Expand Down
19 changes: 11 additions & 8 deletions idl/dsn.layer2.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,17 @@ namespace py pypegasus.replication

struct partition_configuration
{
1:dsn.gpid pid;
2:i64 ballot;
3:i32 max_replica_count;
4:dsn.rpc_address primary;
5:list<dsn.rpc_address> secondaries;
6:list<dsn.rpc_address> last_drops;
7:i64 last_committed_decree;
8:i32 partition_flags;
1:dsn.gpid pid;
2:i64 ballot;
3:i32 max_replica_count;
4:dsn.rpc_address primary;
5:list<dsn.rpc_address> secondaries;
6:list<dsn.rpc_address> last_drops;
7:i64 last_committed_decree;
8:i32 partition_flags;
9:optional dsn.host_port hp_primary;
10:optional list<dsn.host_port> hp_secondaries;
11:optional list<dsn.host_port> hp_last_drops;
}

struct query_cfg_request
Expand Down
1 change: 1 addition & 0 deletions idl/duplication.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ struct duplication_sync_request
1:dsn.rpc_address node;

2:map<dsn.gpid, list<duplication_confirm_entry>> confirm_list;
3:dsn.host_port hp_node;
}

struct duplication_sync_response
Expand Down
52 changes: 30 additions & 22 deletions idl/meta_admin.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,15 +67,16 @@ struct configuration_update_request
{
1:dsn.layer2.app_info info;
2:dsn.layer2.partition_configuration config;
3:config_type type = config_type.CT_INVALID;
4:dsn.rpc_address node;
5:dsn.rpc_address host_node; // deprecated, only used by stateless apps
3:config_type type = config_type.CT_INVALID;
4:dsn.rpc_address node;
5:dsn.rpc_address host_node; // deprecated, only used by stateless apps

// Used for partition split
// if replica is splitting (whose split_status is not NOT_SPLIT)
// the `meta_split_status` will be set
// only used when on_config_sync
6:optional metadata.split_status meta_split_status;
6:optional metadata.split_status meta_split_status;
7:optional dsn.host_port hp_node;
}

// meta server (config mgr) => primary | secondary (downgrade) (w/ new config)
Expand All @@ -100,9 +101,10 @@ struct replica_server_info

struct configuration_query_by_node_request
{
1:dsn.rpc_address node;
1:dsn.rpc_address node;
2:optional list<metadata.replica_info> stored_replicas;
3:optional replica_server_info info;
3:optional replica_server_info info;
4:optional dsn.host_port hp_node;
}

struct configuration_query_by_node_response
Expand All @@ -114,9 +116,10 @@ struct configuration_query_by_node_response

struct configuration_recovery_request
{
1:list<dsn.rpc_address> recovery_set;
2:bool skip_bad_nodes;
3:bool skip_lost_partitions;
1:list<dsn.rpc_address> recovery_set;
2:bool skip_bad_nodes;
3:bool skip_lost_partitions;
4:optional list<dsn.host_port> hp_recovery_set;
}

struct configuration_recovery_response
Expand Down Expand Up @@ -204,7 +207,8 @@ struct configuration_list_apps_response

struct query_app_info_request
{
1:dsn.rpc_address meta_server;
1:dsn.rpc_address meta_server;
2:optional dsn.host_port hp_meta_server;
}

struct query_app_info_response
Expand Down Expand Up @@ -278,8 +282,9 @@ struct query_app_manual_compact_response

struct node_info
{
1:node_status status = node_status.NS_INVALID;
2:dsn.rpc_address address;
1:node_status status = node_status.NS_INVALID;
2:dsn.rpc_address address;
3:optional dsn.host_port hp_address;
}

struct configuration_list_nodes_request
Expand Down Expand Up @@ -342,13 +347,15 @@ enum balancer_request_type

struct configuration_proposal_action
{
1:dsn.rpc_address target;
2:dsn.rpc_address node;
3:config_type type;
1:dsn.rpc_address target;
2:dsn.rpc_address node;
3:config_type type;

// depricated now
// new fields of this struct should start with 5
// 4:i64 period_ts;
5:optional dsn.host_port hp_target;
6:optional dsn.host_port hp_node;
}

struct configuration_balancer_request
Expand All @@ -374,13 +381,14 @@ struct ddd_diagnose_request

struct ddd_node_info
{
1:dsn.rpc_address node;
2:i64 drop_time_ms;
3:bool is_alive; // if the node is alive now
4:bool is_collected; // if replicas has been collected from this node
5:i64 ballot; // collected && ballot == -1 means replica not exist on this node
6:i64 last_committed_decree;
7:i64 last_prepared_decree;
1:dsn.rpc_address node;
2:i64 drop_time_ms;
3:bool is_alive; // if the node is alive now
4:bool is_collected; // if replicas has been collected from this node
5:i64 ballot; // collected && ballot == -1 means replica not exist on this node
6:i64 last_committed_decree;
7:i64 last_prepared_decree;
8:optional dsn.host_port hp_node;
}

struct ddd_partition_info
Expand Down
15 changes: 8 additions & 7 deletions idl/metadata.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,20 +84,21 @@ struct file_meta

struct replica_configuration
{
1:dsn.gpid pid;
2:i64 ballot;
3:dsn.rpc_address primary;
4:partition_status status = partition_status.PS_INVALID;
5:i64 learner_signature;
1:dsn.gpid pid;
2:i64 ballot;
3:dsn.rpc_address primary;
4:partition_status status = partition_status.PS_INVALID;
5:i64 learner_signature;
// Used for bulk load
// secondary will pop all committed mutations even if buffer is not full
6:optional bool pop_all = false;
6:optional bool pop_all = false;
// Used for partition split when primary send prepare message to secondary
// 1. true - secondary should copy mutation in this prepare message synchronously,
// and _is_sync_to_child in mutation structure should set true
// 2. false - secondary copy mutation in this prepare message asynchronously
// NOTICE: it should always be false when update_local_configuration
7:optional bool split_sync_to_child = false;
7:optional bool split_sync_to_child = false;
8:optional dsn.host_port hp_primary;
}

struct replica_info
Expand Down
27 changes: 15 additions & 12 deletions idl/partition_split.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,11 @@ struct query_split_response
// child to primary parent, notifying that itself has caught up with parent
struct notify_catch_up_request
{
1:dsn.gpid parent_gpid;
2:dsn.gpid child_gpid;
3:i64 child_ballot;
4:dsn.rpc_address child_address;
1:dsn.gpid parent_gpid;
2:dsn.gpid child_gpid;
3:i64 child_ballot;
4:dsn.rpc_address child;
5:optional dsn.host_port hp_child;
}

struct notify_cacth_up_response
Expand All @@ -112,10 +113,11 @@ struct notify_cacth_up_response
// primary parent -> child replicas to update partition count
struct update_child_group_partition_count_request
{
1:dsn.rpc_address target_address;
2:i32 new_partition_count;
3:dsn.gpid child_pid;
4:i64 ballot;
1:dsn.rpc_address target;
2:i32 new_partition_count;
3:dsn.gpid child_pid;
4:i64 ballot;
5:optional dsn.host_port hp_target;
}

struct update_child_group_partition_count_response
Expand All @@ -129,10 +131,11 @@ struct update_child_group_partition_count_response
// primary parent -> meta server, register child on meta_server
struct register_child_request
{
1:dsn.layer2.app_info app;
2:dsn.layer2.partition_configuration parent_config;
3:dsn.layer2.partition_configuration child_config;
4:dsn.rpc_address primary_address;
1:dsn.layer2.app_info app;
2:dsn.layer2.partition_configuration parent_config;
3:dsn.layer2.partition_configuration child_config;
4:dsn.rpc_address primary;
5:optional dsn.host_port hp_primary;
}

struct register_child_response
Expand Down
5 changes: 3 additions & 2 deletions idl/replica_admin.thrift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,9 @@ struct disk_info
// This request is sent from client to replica_server.
struct query_disk_info_request
{
1:dsn.rpc_address node;
2:string app_name;
1:dsn.rpc_address node;
2:string app_name;
3:optional dsn.host_port hp_node;
}

// This response is from replica_server to client.
Expand Down
22 changes: 20 additions & 2 deletions run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ export REPORT_DIR="$ROOT/test_report"
export THIRDPARTY_ROOT=$ROOT/thirdparty
export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:${ROOT}/lib:${BUILD_LATEST_DIR}/output/lib:${THIRDPARTY_ROOT}/output/lib:${LD_LIBRARY_PATH}
# Disable AddressSanitizerOneDefinitionRuleViolation, see https://github.com/google/sanitizers/issues/1017 for details.
export ASAN_OPTIONS=detect_odr_violation=0
# Add parameters in order to be able to generate coredump file when run ASAN tests
export ASAN_OPTIONS=detect_odr_violation=0:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
# See https://github.com/gperftools/gperftools/wiki/gperftools'-stacktrace-capturing-methods-and-their-issues.
# Now we choose libgcc, because of https://github.com/apache/incubator-pegasus/issues/1685.
export TCMALLOC_STACKTRACE_METHOD=libgcc # Can be generic_fp, generic_fp_unsafe, libunwind or libgcc
Expand Down Expand Up @@ -241,6 +242,7 @@ function run_build()

if [ ! -z "${SANITIZER}" ]; then
CMAKE_OPTIONS="${CMAKE_OPTIONS} -DSANITIZER=${SANITIZER}"
echo "ASAN_OPTIONS=$ASAN_OPTIONS"
fi

MAKE_OPTIONS="-j$JOB_NUM"
Expand Down Expand Up @@ -484,7 +486,9 @@ function run_test()
# Update options if needed, this should be done before starting onebox to make new options take effect.
if [ "${module}" == "recovery_test" ]; then
master_count=1
opts="meta_state_service_type=meta_state_service_simple;distributed_lock_service_type=distributed_lock_service_simple"
# all test case in recovery_test just run one meta_server, so we should change it
fqdn=`hostname -f`
opts="server_list=$fqdn:34601;meta_state_service_type=meta_state_service_simple;distributed_lock_service_type=distributed_lock_service_simple"
fi
if [ "${module}" == "backup_restore_test" ]; then
opts="cold_backup_disabled=false;cold_backup_checkpoint_reserve_minutes=0;cold_backup_root=onebox"
Expand All @@ -509,6 +513,20 @@ function run_test()

# Run server test.
pushd ${BUILD_LATEST_DIR}/bin/${module}
local function_tests=(
backup_restore_test
recovery_test
restore_test
base_api_test
throttle_test
bulk_load_test
detect_hotspot_test
partition_split_test
)
# function_tests need client used meta_server_list to connect
if [[ "${function_tests[@]}" =~ "${module}" ]]; then
sed -i "s/@LOCAL_HOSTNAME@/${LOCAL_HOSTNAME}/g" ./config.ini
fi
REPORT_DIR=${REPORT_DIR} TEST_BIN=${module} TEST_OPTS=${test_opts} ./run.sh
if [ $? != 0 ]; then
echo "run test \"$module\" in `pwd` failed"
Expand Down
2 changes: 1 addition & 1 deletion scripts/recompile_thrift.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ rm -rf $TMP_DIR
mkdir -p $TMP_DIR
$THIRDPARTY_ROOT/output/bin/thrift --gen cpp:moveable_types -out $TMP_DIR ../idl/rrdb.thrift

sed 's/#include "dsn_types.h"/#include "runtime\/rpc\/rpc_address.h"\n#include "runtime\/task\/task_code.h"\n#include "utils\/blob.h"/' $TMP_DIR/rrdb_types.h > ../src/include/rrdb/rrdb_types.h
sed 's/#include "dsn_types.h"/#include "runtime\/rpc\/rpc_address.h"\n#include "runtime\/rpc\/rpc_host_port.h"\n#include "runtime\/task\/task_code.h"\n#include "utils\/blob.h"/' $TMP_DIR/rrdb_types.h > ../src/include/rrdb/rrdb_types.h
sed 's/#include "rrdb_types.h"/#include <rrdb\/rrdb_types.h>/' $TMP_DIR/rrdb_types.cpp > ../src/base/rrdb_types.cpp

rm -rf $TMP_DIR
Expand Down
Loading

0 comments on commit 5ddb8a1

Please sign in to comment.