diff --git a/Dockerfile b/Dockerfile index d7a46e06f..e232b9103 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,5 @@ # syntax=docker/dockerfile:1 -FROM coinor/cylp -RUN ln -s /usr/bin/python3 /usr/bin/python && ln -s /usr/bin/pip3 /usr/bin/pip -RUN --mount=type=cache,target=/root/.cache/pip pip3 install cvxpy ray numpy pandas tqdm matplotlib graphviz - -# # install CoinOR -# ARG DEBIAN_FRONTEND="noninteractive" -# ENV TZ="America/Los_Angeles" -# RUN --mount=type=cache,target=/var/cache/apt apt update \ -# && apt-get install --no-install-recommends -y git wget ca-certificates build-essential gcc g++ gfortran pkg-config libblas-dev liblapack-dev coinor-libcbc-dev libz-dev \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* -# RUN --mount=type=cache,target=/root/.cache/pip pip install numpy && pip install cylp +FROM python:3.10-slim # increase number of open files and concurrent TCP connections RUN (echo 'net.ipv4.ip_local_port_range = 12000 65535' >> /etc/sysctl.conf) \ @@ -23,7 +12,7 @@ RUN (echo 'net.ipv4.ip_local_port_range = 12000 65535' >> /etc/sysctl.conf) \ # install apt packages RUN --mount=type=cache,target=/var/cache/apt apt update \ - && apt-get install --no-install-recommends -y git wget ca-certificates build-essential rsync graphviz \ + && apt-get install --no-install-recommends -y git wget ca-certificates build-essential graphviz \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index d065ba9af..9424fa1a4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Skylark is composed of two components: A ReplicatorClient that runs locally on y This package represents both components as a single binary. Docker builds a single container with the GatewayDaemon and pushes it to the Github Container Registry (ghcr.io). After provisioning an instance, a GatewayDaemon is started by launching that container. Therefore, it's simple and fast to launch a new Gateway. ### Requirements -* Python 3.8 or greater +* Python 3.7 or greater * Docker * **Ensure you have authenticated your Github account with Docker**: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry * TLDR: diff --git a/scripts/ray_cluster/README.md b/scripts/ray_cluster/README.md deleted file mode 100644 index 346147cc8..000000000 --- a/scripts/ray_cluster/README.md +++ /dev/null @@ -1,12 +0,0 @@ -## Ray Set-up Instructions - -- Make sure you update the latest docker image in the `config.yaml` -- Make sure you have the AWS key-pair as defined in the `config.yaml` - -To launch the Ray Autoscaler cluster: `time ray up -y config.yaml` - -To attach to the head: `ray attach config.yaml` - -To run a python script `script.py`: `ray submit config.yaml script.py` - -To teardown the system: `ray down -y config.yaml` diff --git a/scripts/ray_cluster/script.py b/scripts/ray_cluster/script.py deleted file mode 100644 index 1a2751c5e..000000000 --- a/scripts/ray_cluster/script.py +++ /dev/null @@ -1,31 +0,0 @@ -from collections import Counter -import socket -import time - -import ray - -ray.init(address="auto") - -print( - """This cluster consists of - {} nodes in total - {} CPU resources in total -""".format( - len(ray.nodes()), ray.cluster_resources()["CPU"] - ) -) - - -@ray.remote -def f(): - time.sleep(0.005) - # Return IP address. - return socket.gethostbyname(socket.gethostname()) - - -object_ids = [f.remote() for _ in range(10000)] -ip_addresses = ray.get(object_ids) - -print("Tasks executed") -for ip_address, num_tasks in Counter(ip_addresses).items(): - print(" {} tasks on {}".format(num_tasks, ip_address)) diff --git a/setup.py b/setup.py index 52f3f63c4..c8035a646 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ name="skylark", version="0.1", packages=["skylark"], - python_requires=">=3.8", + python_requires=">=3.7", install_requires=[ "awscrt", "azure-identity", @@ -43,9 +43,9 @@ "matplotlib", "pandas", "questionary", - "ray", "typer", ], + "experiments": ["ray"], "test": ["black", "ipython", "jupyter_console", "pytest", "pytype"], }, entry_points={"console_scripts": ["skylark=skylark.cli.cli:app"]}, diff --git a/skylark/compute/gcp/gcp_server.py b/skylark/compute/gcp/gcp_server.py index e90fbb782..36b4b7055 100644 --- a/skylark/compute/gcp/gcp_server.py +++ b/skylark/compute/gcp/gcp_server.py @@ -33,7 +33,7 @@ def __init__( def uuid(self): return f"{self.region_tag}:{self.gcp_instance_name}" - @lru_cache + @lru_cache(maxsize=1) def get_gcp_instance(self): instances = self.auth.get_gcp_instances(self.gcp_region) if "items" in instances: diff --git a/skylark/compute/utils.py b/skylark/compute/utils.py index 20b95083f..5c90490cb 100644 --- a/skylark/compute/utils.py +++ b/skylark/compute/utils.py @@ -3,7 +3,7 @@ from skylark.utils import logger -@lru_cache +@lru_cache(maxsize=1) def query_which_cloud() -> str: if ( subprocess.call(