From f1e0043bd9f524ac3830a8d4bce33f55ba23f59a Mon Sep 17 00:00:00 2001 From: Sanket Kedia Date: Mon, 8 Feb 2021 10:17:52 +0530 Subject: [PATCH] [#3565] tools: Allow developers to run yugabyted from the repo Summary: Currently, yugabyted can only be run after downloading the tar/gz. Running it from within the repo isn't possible. This fix enables running yugabyted in a build env. The support for '--ui=true' has not been done in this diff fix. Based on discussions with Sanketh, it is more involved and requires some env setup and we need to point sbt to a diff conf file. Hence it is being tracked by a separate issue#7062. Test Plan: Tested by running the runner script as well as locally (Mac). Plan to test it on my dev-server on CentOS. Reviewers: bogdan, wesley, sanketh Reviewed By: sanketh Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D10532 --- bin/yugabyted | 104 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 12 deletions(-) diff --git a/bin/yugabyted b/bin/yugabyted index 0b20603c179b..cec095ace251 100755 --- a/bin/yugabyted +++ b/bin/yugabyted @@ -57,7 +57,6 @@ yugabyte # Script constants. SCRIPT_NAME = os.path.basename(__file__) YUGABYTE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -BIN_DIR = os.path.join(YUGABYTE_DIR, "bin") DEFAULT_CALLHOME = True TRUE_CHOICES = ["true", "True", "t", "T", "yes", "Yes", "y", "Y", "1"] FALSE_CHOICES = ["false", "False", "f", "F", "no", "No", "n", "N", "0"] @@ -124,6 +123,63 @@ TS_MASTER_ADDRS_FLAG = "tserver_master_addrs" start_time_sec = time.time() +# Finds the path where a particular file is present from +# amongst the supplied paths. +def search_file_in_paths(dir_candidates, file_name): + for candidate in dir_candidates: + if os.path.exists(os.path.join(candidate, file_name)): + Output.log("Found directory {} for" + " file {}".format(candidate, file_name)) + return os.path.join(candidate, file_name) + + Output.log_error_and_exit( + "Yugabyte file not found in paths. Please check " + "the paths." + ) + +# Finds the path of a particular YB binary +def find_binary_location(binary_name): + # Default if tar is downloaded + dir_candidates = [ + os.path.join(YUGABYTE_DIR, "bin") + ] + + # Development environment + dir_candidates += [ + os.path.join(YUGABYTE_DIR, "build", "latest", "bin"), + os.path.join(YUGABYTE_DIR, "build-support") + ] + + return search_file_in_paths(dir_candidates, binary_name) + +# Finds the path of the sample data +def find_sample_data_location(data_file): + # Default if tar is downloaded + dir_candidates = [ + os.path.join(YUGABYTE_DIR, "share") + ] + + # Development environment + dir_candidates += [ + os.path.join(YUGABYTE_DIR, "sample") + ] + + return search_file_in_paths(dir_candidates, data_file) + +# Finds the path of the version_metadata.json file +def find_version_metadata_location(version_file): + # Default if tar is downloaded + dir_candidates = [ + os.path.join(YUGABYTE_DIR) + ] + + # Development environment + dir_candidates += [ + os.path.join(YUGABYTE_DIR, "build", "latest") + ] + + return search_file_in_paths(dir_candidates, version_file) + class ControlScript(object): def __init__(self): self.configs = None @@ -211,6 +267,8 @@ class ControlScript(object): # Prints YugabyteDB version. def version(self): + VERSION_METADATA_PATH = find_version_metadata_location("version_metadata.json") + print(VERSION_METADATA_PATH) with open(VERSION_METADATA_PATH) as metadata: data = json.load(metadata) title = "Version".format(SCRIPT_NAME) @@ -264,6 +322,7 @@ class ControlScript(object): "Demo is already running. Concurrent demos are currently unsupported.") # TODO: Race condition currently exists when running demo too close to each other. This # will be solved when concurrent isolated demos are implemented. + Output.print_out("Now creating demo database") self.create_demo() # Ignore kill SIGINT to match normal ysqlsh and psql behavior. @@ -298,7 +357,7 @@ class ControlScript(object): Output.log("Populating {} with sample data...".format(db_name)) files = [] for name in Configs.get_demo_info()[demo_db]["files"]: - files.append(os.path.join(YUGABYTE_DIR, "share", name)) + files.append(os.path.join(find_sample_data_location(name))) ysql_proxy.load_files(files, db=db_name) msg = "Successfully loaded sample database!" @@ -462,7 +521,8 @@ class ControlScript(object): self.configs.saved_data.get("universe_uuid"))) - yb_master_cmd = [os.path.join(BIN_DIR, "yb-master")] + common_gflags + \ + yb_master_cmd = [find_binary_location("yb-master")] + \ + common_gflags + \ [ "--rpc_bind_addresses={}:{}".format(advertise_ip, master_rpc_port), "--server_broadcast_addresses={}:{}".format(advertise_ip, master_rpc_port), @@ -484,7 +544,7 @@ class ControlScript(object): ["--{}".format(flag) for flag in \ self.configs.saved_data.get("master_flags").split(",")]) - yb_tserver_cmd = [os.path.join(BIN_DIR, "yb-tserver")] + common_gflags + \ + yb_tserver_cmd = [find_binary_location("yb-tserver")] + common_gflags + \ [ "--{}={}".format(TS_MASTER_ADDRS_FLAG, master_addresses), "--rpc_bind_addresses={}:{}".format(bind_ip, tserver_rpc_port), @@ -800,7 +860,7 @@ class ControlScript(object): if not sys.platform.startswith('linux'): return - post_install_script_path = os.path.join(YUGABYTE_DIR, 'bin', 'post_install.sh') + post_install_script_path = find_binary_location('post_install.sh') Output.log("Running the post-installation script {} (may be a no-op)".format( post_install_script_path)) @@ -840,7 +900,7 @@ class ControlScript(object): def is_yw_initialized(self): # Check Play evolutions table was created. Output.log("Checking play_evolutions table") - list_tables_cmd = [os.path.join(BIN_DIR, "ysqlsh"), "-d", WEBSERVER_DB, "-c", "\d"] + list_tables_cmd = [find_binary_location("ysqlsh"), "-d", WEBSERVER_DB, "-c", "\d"] out, err, ret_code = run_process(list_tables_cmd) Output.log("Finished checking play evolutions table") return not err and not ret_code and "play_evolutions" in out @@ -1142,7 +1202,8 @@ class ControlScript(object): # Parse config file and input args. Validate them and save any new configs. def validate_and_set_configs(self, args): - default_base_dir = os.path.join(YUGABYTE_DIR, "var") + home_dir = os.path.expanduser("~") + default_base_dir = os.path.join(home_dir, "var") default_conf_path = os.path.join(default_base_dir, "conf", "{}.conf".format(SCRIPT_NAME)) base_dir = os.path.abspath(args.base_dir) if args.base_dir else default_base_dir base_dir_conf = '' @@ -1383,6 +1444,10 @@ class ControlScript(object): Output.log("cmd = {} using config file: {} (args.config={})".format( args.parser, self.conf_file, args.config)) + # Initialize the binary path of ybadmin + # TODO(Sanket): Clean up and refactor this file + YBAdminProxy.populate_binary_path() + try: args.func() except Exception as e: @@ -1863,7 +1928,11 @@ class Diagnostics(object): # Proxy for parsing output from yb-admin commands. class YBAdminProxy(object): - path = os.path.join(BIN_DIR, "yb-admin") + path = None + + @staticmethod + def populate_binary_path(): + YBAdminProxy.path = find_binary_location("yb-admin") @staticmethod def add_master(master_addrs, new_master_ip, timeout=10): @@ -1926,20 +1995,29 @@ class YBAdminProxy(object): # Proxy for ysqlsh commands. class YsqlProxy(object): - def __init__(self, ip, port, path=os.path.join(BIN_DIR, YUGABYTE_API_CLIENT_PROGRAMS["ysql"]), + def __init__(self, ip, port, path=None, get_default_credentials=False): + if path is None: + path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ysql"]) self.cmd = [path, "-h", str(ip), "-p", str(port)] self.setup_env_init = EnvBasedCredentials() self.username, self.password, self.db = self.setup_env_init.get_ysql_credentials( get_default_credentials) - self.env = {"PGUSER": self.username, "PGPASSWORD": self.password, "PGDATABASE": self.db} + env_var = os.environ.copy() + env_var["PGUSER"] = self.username + env_var["PGPASSWORD"] = self.password + env_var["PGDATABASE"] = self.db + + self.env = env_var # Starts interactive YSQL shell. def connect(self, db=None): if db is None: db = self.db cmd = self.cmd + ["-d", db] - shell = subprocess.Popen(cmd, env={"PGUSER": self.username}) + env_var = os.environ.copy() + env_var["PGUSER"] = self.username + shell = subprocess.Popen(cmd, env=env_var) shell.communicate() # Checks if db exists. @@ -2015,8 +2093,10 @@ class YsqlProxy(object): # Proxy for ycqlsh commands. class YcqlProxy(object): - def __init__(self, ip, port, path=os.path.join(BIN_DIR, YUGABYTE_API_CLIENT_PROGRAMS["ycql"]), + def __init__(self, ip, port, path=None, get_default_credentials=False): + if path is None: + path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ycql"]) self.cmd = [path, str(ip), str(port)] self.setup_env_init = EnvBasedCredentials() self.username, self.password, self.keyspace = self.setup_env_init.get_ycql_credentials(