[#3565] tools: Allow developers to run yugabyted from the repo

Summary:
Currently, yugabyted can only be run after downloading the tar/gz. Running it from within the repo
isn't possible.

This fix enables running yugabyted in a build env. The support for '--ui=true' has not been done in
this diff fix. Based on discussions with Sanketh, it is more involved and requires some env setup and we need to
point sbt to a diff conf file. Hence it is being tracked by a separate issue#7062.

Test Plan: Tested by running the runner script as well as locally (Mac). Plan to test it on my dev-server on CentOS.

Reviewers: bogdan, wesley, sanketh

Reviewed By: sanketh

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D10532

bin/yugabyted

@@ -57,7 +57,6 @@ yugabyte
 # Script constants.
 SCRIPT_NAME = os.path.basename(__file__)
 YUGABYTE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-BIN_DIR = os.path.join(YUGABYTE_DIR, "bin")
 DEFAULT_CALLHOME = True
 TRUE_CHOICES = ["true", "True", "t", "T", "yes", "Yes", "y", "Y", "1"]
 FALSE_CHOICES = ["false", "False", "f", "F", "no", "No", "n", "N", "0"]
@@ -124,6 +123,63 @@ TS_MASTER_ADDRS_FLAG = "tserver_master_addrs"
 
 start_time_sec = time.time()
 
+# Finds the path where a particular file is present from
+# amongst the supplied paths.
+def search_file_in_paths(dir_candidates, file_name):
+    for candidate in dir_candidates:
+        if os.path.exists(os.path.join(candidate, file_name)):
+            Output.log("Found directory {} for"
+                        " file {}".format(candidate, file_name))
+            return os.path.join(candidate, file_name)
+
+    Output.log_error_and_exit(
+        "Yugabyte file not found in paths. Please check "
+        "the paths."
+    )
+
+# Finds the path of a particular YB binary
+def find_binary_location(binary_name):
+    # Default if tar is downloaded
+    dir_candidates = [
+        os.path.join(YUGABYTE_DIR, "bin")
+    ]
+
+    # Development environment
+    dir_candidates += [
+        os.path.join(YUGABYTE_DIR, "build", "latest", "bin"),
+        os.path.join(YUGABYTE_DIR, "build-support")
+    ]
+
+    return search_file_in_paths(dir_candidates, binary_name)
+
+# Finds the path of the sample data
+def find_sample_data_location(data_file):
+    # Default if tar is downloaded
+    dir_candidates = [
+        os.path.join(YUGABYTE_DIR, "share")
+    ]
+
+    # Development environment
+    dir_candidates += [
+        os.path.join(YUGABYTE_DIR, "sample")
+    ]
+
+    return search_file_in_paths(dir_candidates, data_file)
+
+# Finds the path of the version_metadata.json file
+def find_version_metadata_location(version_file):
+    # Default if tar is downloaded
+    dir_candidates = [
+        os.path.join(YUGABYTE_DIR)
+    ]
+
+    # Development environment
+    dir_candidates += [
+        os.path.join(YUGABYTE_DIR, "build", "latest")
+    ]
+
+    return search_file_in_paths(dir_candidates, version_file)
+
 class ControlScript(object):
     def __init__(self):
         self.configs = None
@@ -211,6 +267,8 @@ class ControlScript(object):
 
     # Prints YugabyteDB version.
     def version(self):
+        VERSION_METADATA_PATH = find_version_metadata_location("version_metadata.json")
+        print(VERSION_METADATA_PATH)
         with open(VERSION_METADATA_PATH) as metadata:
             data = json.load(metadata)
             title = "Version".format(SCRIPT_NAME)
@@ -264,6 +322,7 @@ class ControlScript(object):
                 "Demo is already running. Concurrent demos are currently unsupported.")
         # TODO: Race condition currently exists when running demo too close to each other. This
         # will be solved when concurrent isolated demos are implemented.
+        Output.print_out("Now creating demo database")
         self.create_demo()
 
         # Ignore kill SIGINT to match normal ysqlsh and psql behavior.
@@ -298,7 +357,7 @@ class ControlScript(object):
         Output.log("Populating {} with sample data...".format(db_name))
         files = []
         for name in Configs.get_demo_info()[demo_db]["files"]:
-            files.append(os.path.join(YUGABYTE_DIR, "share", name))
+            files.append(os.path.join(find_sample_data_location(name)))
         ysql_proxy.load_files(files, db=db_name)
 
         msg = "Successfully loaded sample database!"
@@ -462,7 +521,8 @@ class ControlScript(object):
                 self.configs.saved_data.get("universe_uuid")))
 
 
-        yb_master_cmd = [os.path.join(BIN_DIR, "yb-master")] + common_gflags + \
+        yb_master_cmd = [find_binary_location("yb-master")] + \
+            common_gflags + \
             [
                 "--rpc_bind_addresses={}:{}".format(advertise_ip, master_rpc_port),
                 "--server_broadcast_addresses={}:{}".format(advertise_ip, master_rpc_port),
@@ -484,7 +544,7 @@ class ControlScript(object):
                 ["--{}".format(flag) for flag in \
                     self.configs.saved_data.get("master_flags").split(",")])
 
-        yb_tserver_cmd = [os.path.join(BIN_DIR, "yb-tserver")] + common_gflags + \
+        yb_tserver_cmd = [find_binary_location("yb-tserver")] + common_gflags + \
             [
                 "--{}={}".format(TS_MASTER_ADDRS_FLAG, master_addresses),
                 "--rpc_bind_addresses={}:{}".format(bind_ip, tserver_rpc_port),
@@ -800,7 +860,7 @@ class ControlScript(object):
         if not sys.platform.startswith('linux'):
             return
 
-        post_install_script_path = os.path.join(YUGABYTE_DIR, 'bin', 'post_install.sh')
+        post_install_script_path = find_binary_location('post_install.sh')
 
         Output.log("Running the post-installation script {} (may be a no-op)".format(
                     post_install_script_path))
@@ -840,7 +900,7 @@ class ControlScript(object):
     def is_yw_initialized(self):
         # Check Play evolutions table was created.
         Output.log("Checking play_evolutions table")
-        list_tables_cmd = [os.path.join(BIN_DIR, "ysqlsh"), "-d", WEBSERVER_DB, "-c", "\d"]
+        list_tables_cmd = [find_binary_location("ysqlsh"), "-d", WEBSERVER_DB, "-c", "\d"]
         out, err, ret_code = run_process(list_tables_cmd)
         Output.log("Finished checking play evolutions table")
         return not err and not ret_code and "play_evolutions" in out
@@ -1142,7 +1202,8 @@ class ControlScript(object):
     # Parse config file and input args. Validate them and save any new configs.
     def validate_and_set_configs(self, args):
 
-        default_base_dir = os.path.join(YUGABYTE_DIR, "var")
+        home_dir = os.path.expanduser("~")
+        default_base_dir = os.path.join(home_dir, "var")
         default_conf_path = os.path.join(default_base_dir, "conf", "{}.conf".format(SCRIPT_NAME))
         base_dir = os.path.abspath(args.base_dir) if args.base_dir else default_base_dir
         base_dir_conf = ''
@@ -1383,6 +1444,10 @@ class ControlScript(object):
         Output.log("cmd = {} using config file: {} (args.config={})".format(
             args.parser, self.conf_file, args.config))
 
+        # Initialize the binary path of ybadmin
+        # TODO(Sanket): Clean up and refactor this file
+        YBAdminProxy.populate_binary_path()
+
         try:
             args.func()
         except Exception as e:
@@ -1863,7 +1928,11 @@ class Diagnostics(object):
 
 # Proxy for parsing output from yb-admin commands.
 class YBAdminProxy(object):
-    path = os.path.join(BIN_DIR, "yb-admin")
+    path = None
+
+    @staticmethod
+    def populate_binary_path():
+        YBAdminProxy.path = find_binary_location("yb-admin")
 
     @staticmethod
     def add_master(master_addrs, new_master_ip, timeout=10):
@@ -1926,20 +1995,29 @@ class YBAdminProxy(object):
 
 # Proxy for ysqlsh commands.
 class YsqlProxy(object):
-    def __init__(self, ip, port, path=os.path.join(BIN_DIR, YUGABYTE_API_CLIENT_PROGRAMS["ysql"]),
+    def __init__(self, ip, port, path=None,
                     get_default_credentials=False):
+        if path is None:
+            path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ysql"])
         self.cmd = [path, "-h", str(ip), "-p", str(port)]
         self.setup_env_init = EnvBasedCredentials()
         self.username, self.password, self.db = self.setup_env_init.get_ysql_credentials(
                                                     get_default_credentials)
-        self.env = {"PGUSER": self.username, "PGPASSWORD": self.password, "PGDATABASE": self.db}
+        env_var = os.environ.copy()
+        env_var["PGUSER"] = self.username
+        env_var["PGPASSWORD"] = self.password
+        env_var["PGDATABASE"] = self.db
+
+        self.env = env_var
 
     # Starts interactive YSQL shell.
     def connect(self, db=None):
         if db is None:
             db = self.db
         cmd = self.cmd + ["-d", db]
-        shell = subprocess.Popen(cmd, env={"PGUSER": self.username})
+        env_var = os.environ.copy()
+        env_var["PGUSER"] = self.username
+        shell = subprocess.Popen(cmd, env=env_var)
         shell.communicate()
 
     # Checks if db exists.
@@ -2015,8 +2093,10 @@ class YsqlProxy(object):
 
 # Proxy for ycqlsh commands.
 class YcqlProxy(object):
-    def __init__(self, ip, port, path=os.path.join(BIN_DIR, YUGABYTE_API_CLIENT_PROGRAMS["ycql"]),
+    def __init__(self, ip, port, path=None,
                     get_default_credentials=False):
+        if path is None:
+            path = find_binary_location(YUGABYTE_API_CLIENT_PROGRAMS["ycql"])
         self.cmd = [path, str(ip), str(port)]
         self.setup_env_init = EnvBasedCredentials()
         self.username, self.password, self.keyspace = self.setup_env_init.get_ycql_credentials(