Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enhance private cloud deployment support
Browse files Browse the repository at this point in the history
Martin-Molinero committed Nov 20, 2024
1 parent 1b9246e commit f360668
Showing 2 changed files with 43 additions and 18 deletions.
53 changes: 38 additions & 15 deletions lean/commands/private_cloud/start.py
Original file line number Diff line number Diff line change
@@ -48,8 +48,10 @@ def deploy(ip: str, port: int, token: str, slave: bool, update: bool, no_update:
logger.info(f"Starting {compute_node_name}...")
compute_directory = Path(f"~/.lean/compute/{compute_node_name}").expanduser()
lean_config["node-name"] = compute_node_name

run_options = container.lean_runner.get_basic_docker_config_without_algo(lean_config, None, True, None, None,
None, compute_directory)
run_options["environment"]["AIRLOCK"] = compute_directory
run_options["mounts"].append(Mount(target="/QuantConnect/platform-services/airlock",
source=str(compute_directory), type="bind"))
run_options["mounts"].append(Mount(target="/var/run/docker.sock", source="/var/run/docker.sock",
@@ -59,13 +61,24 @@ def deploy(ip: str, port: int, token: str, slave: bool, update: bool, no_update:
type="bind", read_only=True))
container.lean_runner.parse_extra_docker_config(run_options, loads(extra_docker_config))

if not image:
image = "quantconnect/platform-services:latest"

if not slave:
run_options["ports"]["9696"] = str(port)
run_options["ports"]["9697"] = str(get_free_port())

root_directory = container.lean_config_manager.get_cli_root_directory()
run_options["volumes"][str(root_directory)] = {"bind": "/LeanCLIWorkspace", "mode": "rw"}

if not ip.replace('.', '').isnumeric():
labels = {}
for name, value in container.docker_manager.get_image_labels(image):
if slave and name == "slave" or not slave and name == "master":
for key, label in loads(value).items():
labels[key] = label.replace("{{domain}}", ip)
run_options["labels"] = labels

run_options["remove"] = False
run_options["name"] = compute_node_name
run_options["environment"]["MODE"] = str('slave') if slave else str('master')
@@ -173,32 +186,42 @@ def start(master: bool,
lean_config = container.lean_config_manager.get_complete_lean_config(None, None, None)

if master:
master_port_option = f" --master-port {master_port}"
if not master_ip.replace('.', '').isnumeric():
lean_config["domain-label"] = master_ip
master_port_option = ''
deploy(master_ip, master_port, token, False, update, no_update, image, lean_config, extra_docker_config)
if master_port == 0:
master_port = container.docker_manager.get_container_port(COMPUTE_MASTER, "9696/tcp")

logger.info(f"Slaves can be added running: "
f"lean private-cloud start --slave --master-ip {master_ip} --token \"{token}\" --master-port {master_port}")
f"lean private-cloud start --slave --master-ip {master_ip} --token \"{token}\"{master_port_option}")

compute_index = len(get_private_cloud_containers([COMPUTE_SLAVE]))
if compute:
logger.debug(f"Starting given compute configuration: {compute}")

if not slave_ip:
logger.debug(f"'slave-ip' was not given will try to figure it out...")
retry_count = 0
while retry_count < 10:
retry_count += 1
try:
from requests import get
resp = get(f'http://{master_ip}:{master_port}', stream=True)
slave_ip = resp.raw._connection.sock.getsockname()[0]
break
except Exception as e:
from time import sleep
sleep(1)
pass
if master and not master_ip.replace('.', '').isnumeric():
slave_ip = master_ip
if not slave_ip:
logger.debug(f"'slave-ip' was not given will try to figure it out...")
retry_count = 0
while retry_count < 10:
retry_count += 1
try:
from requests import get
resp = get(f'http://{master_ip}:{master_port}', stream=True)
slave_ip = resp.raw._connection.sock.getsockname()[0]
break
except Exception as e:
from time import sleep
sleep(1)
pass
if not slave_ip.replace('.', '').isnumeric():
lean_config["domain-label"] = slave_ip
lean_config["self-ip-address"] = slave_ip
logger.info(f"Using ip address '{slave_ip}' as own")
logger.debug(f"Using address '{slave_ip}' as own")

for configuration in compute:
lean_config["compute"] = configuration
8 changes: 5 additions & 3 deletions lean/components/docker/docker_manager.py
Original file line number Diff line number Diff line change
@@ -38,10 +38,12 @@ def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager:
self._temp_manager = temp_manager
self._platform_manager = platform_manager

def get_image_label(self, image: DockerImage, label: str, default: str) -> str:
docker_image = self._get_docker_client().images.get(str(image))
def get_image_labels(self, image: str) -> str:
docker_image = self._get_docker_client().images.get(image)
return docker_image.labels.items()

for name, value in docker_image.labels.items():
def get_image_label(self, image: DockerImage, label: str, default: str) -> str:
for name, value in self.get_image_labels(str(image)):
if name == label:
self._logger.debug(f"Label '{label}' found in image '{image.name}', value {value}")
return value

0 comments on commit f360668

Please sign in to comment.